00000800:00000200:0.1F:1769074509.739786:0:2360:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006c31bc20] -> 12345-192.168.204.121@tcp (2) 00000800:00000200:3.1F:1769074509.747417:0:2349:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1769074509.747439:0:2349:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.121@tcp ip 192.168.204.121:988 00000800:00000200:3.1:1769074509.747447:0:2349:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000049b0c1dc type 2, nob 96 niov 1 nkiov 0 00000800:00000200:2.1F:1769074509.765309:0:2349:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:2.1:1769074509.796015:0:2349:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000080:00200000:1.0:1769074927.904876:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074927.904893:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074927.904899:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074927.904919:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2898, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074927.904923:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2898, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074927.904937:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2898,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074927.904990:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074927.904999:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074927.905010:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074927.905032:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93501020aa00 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912941184:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074927.905148:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361cc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074927.910033:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93501020aa00 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912941184:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074927.910127:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.910153:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.910162:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.910177:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.910189:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.910204:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.910212:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.910220:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074927.910232:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff93501020aa00 x1855008912941184/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074943 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074927.910245:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074927.910259:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074927.910270:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074927.910278:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074927.910327:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074927.910341:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074927.910346:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d758 00000080:00010000:1.0:1769074927.910357:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074927.910361:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d758 from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074927.910364:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074927.910378:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074927.910412:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074927.910421:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2898, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074927.910455:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074927.910469:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074927.910512:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074927.910519:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074927.910521:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d758 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf9594c expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074927.910548:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074927.910555:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff93501020aa00 x1855008912941440/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074927.910578:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93501020aa00 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912941440:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074927.910666:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361cc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074927.916650:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93501020aa00 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912941440:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074927.916837:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074927.916866:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074927.916895:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.916916:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074927.916923:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074927.916929:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074927.916933:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074927.916941:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074927.916945:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074927.916953:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074927.916970:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074927.916978:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074927.916984:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074927.917002:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2899, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074927.917006:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2899, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074927.917020:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2899,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074927.917112:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0800/0x410246667097d75f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074927.917120:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0800/0x410246667097d75f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074927.917130:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0800/0x410246667097d75f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074927.917153:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93501020a300 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912941568:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074927.917230:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361cc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.038089:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007a3c3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.042981:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935004a0a300 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912944640:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.043018:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.043202:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.043213:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.043227:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.043241:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.043255:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.043264:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.043272:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.043285:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff935004a0a300 x1855008912944640/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.043303:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.043316:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.043328:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.043336:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.043386:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.043401:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.043406:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d797 00000080:00010000:1.0:1769074928.043417:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.043422:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d797 from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.043425:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.043434:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.043473:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.043482:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2907, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.043516:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.043532:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.043578:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.043676:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.043680:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097d797 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b05 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.044662:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.044674:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff93501020a300 x1855008912944896/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.044700:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93501020a300 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912944896:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.045027:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.052400:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93501020a300 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912944896:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.052509:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.052536:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.052558:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.052574:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.052583:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.052599:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.052603:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.052612:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.052616:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.052625:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.052642:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.052649:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.052656:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.052673:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2908, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.052677:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2908, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.052692:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2908,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.053100:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.053109:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.053120:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.053144:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935010208700 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945024:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.053224:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.059136:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935010208700 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945024:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.059169:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.059196:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.059204:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.059217:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.059227:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.059241:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.059249:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.059257:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.059269:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff935010208700 x1855008912945024/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.059282:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.059297:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.059309:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.059317:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.059367:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.059382:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.059387:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d79e 00000080:00010000:1.0:1769074928.059398:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.059402:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d79e from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.059405:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.059418:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.059452:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.059460:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2908, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.059494:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.059510:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.059556:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.059562:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.059565:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d79e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b36 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.059605:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.059613:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff935010208700 x1855008912945280/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.059638:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935010208700 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945280:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.060154:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.065581:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935010208700 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945280:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.065682:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.065717:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.065738:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.065758:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.065766:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.065771:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.065775:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.065783:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.065787:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.065795:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.065814:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.065821:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.065826:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.065854:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2909, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.065859:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2909, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.065873:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2909,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.065938:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.065945:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.065955:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.065977:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935010209500 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945408:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.066093:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.071310:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935010209500 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945408:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.071342:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.071369:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.071378:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.071391:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.071402:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.071414:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.071423:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.071430:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.071441:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff935010209500 x1855008912945408/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.071460:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.071474:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.071484:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.071492:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.071541:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.071554:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.071558:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d7a5 00000080:00010000:1.0:1769074928.071569:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.071573:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d7a5 from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.071575:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.071594:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.071629:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.071637:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2909, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.071668:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.071682:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.071724:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.071731:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.071734:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7a5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b67 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.071759:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.071767:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff935010209500 x1855008912945664/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.071791:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935010209500 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945664:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.071903:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.077409:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935010209500 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945664:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.077515:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.077540:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.077560:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.077573:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.077580:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.077594:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.077597:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.077604:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.077606:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.077613:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.077626:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.077634:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.077639:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.077653:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2910, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.077657:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2910, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.077667:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2910,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.077717:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.077722:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.077731:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.077750:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945792:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.077823:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.083139:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912945792:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.083174:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.083200:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.083209:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.083223:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.083235:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.083248:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.083256:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.083263:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.083275:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a7100 x1855008912945792/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.083287:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.083301:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.083312:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.083320:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.083370:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.083384:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.083390:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d7ac 00000080:00010000:1.0:1769074928.083401:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.083405:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d7ac from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.083408:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.083421:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.083457:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.083464:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2910, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.083496:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.083512:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.083556:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.083564:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.083567:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d7ac lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95b98 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.083603:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.083611:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008912946048/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.083635:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946048:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.083732:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.089166:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946048:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.089273:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.089300:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.089322:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.089343:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.089352:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.089358:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.089362:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.089369:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.089374:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.089383:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.089400:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.089407:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.089413:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.089429:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2911, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.089434:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2911, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.089447:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2911,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.089500:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.089507:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.089517:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.089538:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946176:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.089619:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.097352:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946176:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.097387:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.097417:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.097426:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.097439:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.097451:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.097464:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.097473:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.097481:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.097494:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a4000 x1855008912946176/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.097513:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.097528:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.097540:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.097548:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.097611:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.097627:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.097633:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d7b3 00000080:00010000:1.0:1769074928.097645:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.097649:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d7b3 from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.097652:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.097661:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.097700:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.097721:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2911, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.097758:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.097773:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.097823:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.097830:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.097834:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7b3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95bc9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.097864:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.097872:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4000 x1855008912946432/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.097907:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946432:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.098011:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.104648:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946432:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.104764:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.104790:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.104813:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.104830:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.104838:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.104844:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.104848:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.104856:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.104860:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.104868:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.104894:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.104904:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.104910:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.104931:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2912, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.104936:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2912, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.104950:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2912,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.105004:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7ba lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.105011:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7ba lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.105040:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7ba lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.105135:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946560:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.105218:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.113654:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912946560:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.113691:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d7ba lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95bfa expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074928.359508:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007a3c9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.369524:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912951808:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.370002:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.370031:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.370688:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.370710:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.370720:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.370729:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.370734:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.370751:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.370756:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.370765:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.370783:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.370794:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.370800:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.370827:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2926, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.370842:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2926, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.370858:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2926,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.370933:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.370940:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.370950:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.370977:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912951936:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.371108:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.379228:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912951936:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.379263:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.379292:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.379300:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.379314:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.379327:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.379340:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.379348:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.379357:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.379368:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a4380 x1855008912951936/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.379380:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.379398:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.379409:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.379415:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.379468:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.379484:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.379488:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d81c 00000080:00010000:1.0:1769074928.379501:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.379504:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d81c from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.379508:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.379517:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.379551:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.379560:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2926, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.379681:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.379697:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.380028:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.380036:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.380039:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d81c lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95ea8 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.380104:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.380112:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4380 x1855008912952192/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.380138:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952192:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.380324:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.386353:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952192:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.386456:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.386484:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.386508:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.386523:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.386531:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.386537:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.386541:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.386550:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.386554:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.386562:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.386580:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.386588:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.386594:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.386612:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2927, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.386617:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2927, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.386631:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2927,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.386680:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.386688:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.386699:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.387956:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952320:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.388192:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.396402:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952320:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.396437:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.396468:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.396477:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.396490:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.396503:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.396516:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.396525:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.396532:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.396543:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a5880 x1855008912952320/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.396554:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.396569:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.396668:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.396677:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.396997:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.397015:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.397022:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d823 00000080:00010000:1.0:1769074928.397034:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.397038:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d823 from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.397043:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.397092:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.397133:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.397143:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2927, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.397179:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.397196:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.397247:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.397254:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.397257:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1600/0x410246667097d823 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95ed9 expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.397285:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.397294:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5880 x1855008912952576/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.397318:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952576:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.397701:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.407664:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952576:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.407788:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.407818:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.407841:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.407862:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.407870:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.407887:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.407891:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.407899:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.407903:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.407913:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.407931:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.407938:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.407944:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.407965:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2928, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.407969:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2928, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.407983:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2928,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.408101:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.408109:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.408121:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.408144:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952704:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.408245:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.413377:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952704:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.413414:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.413441:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.413450:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.413464:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.413475:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.413488:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.413497:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.413504:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.413516:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a6d80 x1855008912952704/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.413533:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.413546:0:21373:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.413555:0:21373:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.413563:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.413625:0:21373:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff935011535348): rc = 0 00000080:00200000:1.0:1769074928.413641:0:21373:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074928.413646:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) for lock 0x410246667097d82a 00000080:00010000:1.0:1769074928.413657:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff935011535348) 00000080:00010000:1.0:1769074928.413661:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d82a from it ffffa8e541cb3e00 00010000:00010000:1.0:1769074928.413665:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.413673:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.413720:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.413730:0:21373:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile2928, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8) 00010000:00010000:1.0:1769074928.413775:0:21373:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.413791:0:21373:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.414235:0:21373:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.414245:0:21373:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.414248:0:21373:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d82a lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95f0a expref: -99 pid: 21373 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074928.414277:0:21373:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff935006f53800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.414285:0:21373:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6d80 x1855008912952960/t0(0) o36->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.414311:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952960:192.168.204.121@tcp:36:unlinkmany.0 00000100:00100000:1.0:1769074928.414420:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.422174:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912952960:192.168.204.121@tcp:36:unlinkmany.0 00000080:00200000:1.0:1769074928.422280:0:21373:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074928.422309:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.422330:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.422345:0:21373:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.422353:0:21373:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.422358:0:21373:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074928.422362:0:21373:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667096e759 00000080:00010000:1.0:1769074928.422370:0:21373:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e759 from it ffffa8e541cb3c10 00010000:00010000:1.0:1769074928.422374:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.422384:0:21373:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.422402:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.422410:0:21373:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=24 00000080:00200000:1.0:1769074928.422415:0:21373:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115ac2c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.422435:0:21373:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile2929, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), flags=0 00000080:00200000:1.0:1769074928.422439:0:21373:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile2929, dir=[0x200000401:0x20:0x0](ffff9350115ac2c8), intent=getattr 00000002:00010000:1.0:1769074928.422451:0:21373:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile2929,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.422504:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.422512:0:21373:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00010000:00010000:1.0:1769074928.422523:0:21373:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21373 initiator: MDT0 00000100:00100000:1.0:1769074928.422545:0:21373:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912953088:192.168.204.121@tcp:101:unlinkmany.0 00000100:00100000:1.0:1769074928.422705:0:21373:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.430692:0:21373:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job unlinkmany:84af2899-c956-435d-9e3c-5d5fbc3c9246:21373:1855008912953088:192.168.204.121@tcp:101:unlinkmany.0 00010000:00010000:1.0:1769074928.430958:0:21373:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.430987:0:21373:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.430995:0:21373:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.431008:0:21373:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.431021:0:21373:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.431035:0:21373:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.431044:0:21373:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.431184:0:21373:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.431197:0:21373:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a6a00 x1855008912953088/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'unlinkmany.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.431208:0:21373:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0400/0x410246667097d831 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95f3b expref: -99 pid: 21373 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074928.474142:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912953984:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074928.474178:0:21385:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474200:0:21385:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474210:0:21385:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474224:0:21385:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474243:0:21385:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474257:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474265:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474272:0:21385:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.474284:0:21385:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff935002a63480 x1855008912953984/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.474297:0:21385:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474307:0:21385:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/32 (0 0) 00000080:00200000:1.0:1769074928.474314:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074928.474338:0:21385:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x20:0x0], updating i_size 120320 00000080:00010000:1.0:1769074928.474344:0:21385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x20:0x0](ffff9350115aba88) for lock 0x410246667097d846 00000080:00010000:1.0:1769074928.474353:0:21385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x20:0x0](ffff9350115aba88) 00000080:00010000:1.0:1769074928.474356:0:21385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d846 from it ffffa8e541cdbcb8 00010000:00010000:1.0:1769074928.474359:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474367:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.474476:0:21385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074928.474499:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.474509:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474522:0:21385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.474529:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.474534:0:21385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074928.474538:0:21385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667096e5a7 00000080:00010000:1.0:1769074928.474542:0:21385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e5a7 from it ffffa8e541cdbbf8 00010000:00010000:1.0:1769074928.474545:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.474551:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.474558:0:21385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.474566:0:21385:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=266 00000080:00200000:1.0:1769074928.474573:0:21385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074928.474971:0:21385:0:(file.c:1001:ll_file_open()) START file lockdir:[0x200000401:0x20:0x0](ffff9350115aba88), flags 704400 00000002:00010000:1.0:1769074928.474996:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x20:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240704401 00010000:00010000:1.0:1769074928.475096:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.475105:0:21385:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00000020:00100000:1.0:1769074928.475123:0:21385:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074928.475132:0:21385:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7480 x1855008912954240/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074928.475141:0:21385:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00000100:00100000:1.0:1769074928.475165:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954240:192.168.204.121@tcp:101:rm.0 00000100:00100000:1.0:1769074928.475446:0:21385:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.479908:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954240:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074928.479940:0:21385:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.479954:0:21385:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.479961:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.479968:0:21385:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.479976:0:21385:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.479981:0:21385:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074928.479986:0:21385:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.479993:0:21385:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.480004:0:21385:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.480011:0:21385:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074928.480019:0:21385:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.480024:0:21385:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d84d lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21385 initiator: MDT0 00000002:00100000:1.0:1769074928.480045:0:21385:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9350062a7480 x1855008912954240/t4294973337(4294973337) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074928.480100:0:21385:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7480 x1855008912954240/t4294973337(4294973337) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074928.480114:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074928.480139:0:21385:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x20:0x0], updating i_size 120320 00000002:00100000:1.0:1769074928.480158:0:21385:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7480 x1855008912954240/t4294973337(4294973337) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1769074928.480233:0:21385:0:(file.c:6252:ll_getattr_dentry()) START file lockdir:[0x200000401:0x20:0x0](ffff9350115aba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074928.480242:0:21385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88),name=lockdir 00000002:00010000:1.0:1769074928.480264:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x20:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.480282:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.480296:0:21385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.480304:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.480311:0:21385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x20:0x0](ffff9350115aba88) 00000080:00010000:1.0:1769074928.480315:0:21385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x20:0x0](ffff9350115aba88) for lock 0x410246667097d846 00000080:00010000:1.0:1769074928.480320:0:21385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d846 from it ffffa8e541cdbcd8 00010000:00010000:1.0:1769074928.480324:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.480333:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.480437:0:21385:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88) pos/size0/120320 32bit_api 0 00000002:00010000:1.0:1769074928.480452:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x20:0x0]) in obj [0x200000401:0x20:0x0], intent: readdir flags 0 00010000:00010000:1.0:1769074928.480459:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.480466:0:21385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074928.481180:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954368:192.168.204.121@tcp:37:rm.0 00000100:00100000:1.0:1769074928.481302:0:21385:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.489929:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954368:192.168.204.121@tcp:37:rm.0 00010000:00010000:1.0:1769074928.491286:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.491310:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074928.491488:0:21385:0:(file.c:424:ll_file_release()) START file lockdir:[0x200000401:0x20:0x0](ffff9350115aba88), flags 704000 00000002:00100000:1.0:1769074928.491576:0:21385:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350062a7480 x1855008912954240/t4294973337(4294973337) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074928.491603:0:21385:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074928.491609:0:21385:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7800 x1855008912954624/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074928.491636:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954624:192.168.204.121@tcp:35:rm.0 00000100:00100000:1.0:1769074928.491754:0:21385:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.499958:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954624:192.168.204.121@tcp:35:rm.0 00000002:00100000:1.0:1769074928.500004:0:21385:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9350062a7480 x1855008912954240/t4294973337(4294973337) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074928.500146:0:21385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074928.500171:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.500195:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.500211:0:21385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.500219:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.500226:0:21385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074928.500229:0:21385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667096e5a7 00000080:00010000:1.0:1769074928.500237:0:21385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667096e5a7 from it ffffa8e541cdbbf8 00010000:00010000:1.0:1769074928.500241:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.500251:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667096e5a7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3ee9d expref: -99 pid: 21342 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.500273:0:21385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074928.500282:0:21385:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lockdir, flags=266 00000080:00200000:1.0:1769074928.500289:0:21385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074928.500299:0:21385:0:(file.c:1001:ll_file_open()) START file lockdir:[0x200000401:0x20:0x0](ffff9350115aba88), flags 704400 00000002:00010000:1.0:1769074928.500316:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x20:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240704401 00010000:00010000:1.0:1769074928.500375:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.500383:0:21385:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00000020:00100000:1.0:1769074928.500395:0:21385:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074928.500401:0:21385:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7800 x1855008912954880/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074928.500410:0:21385:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00000100:00100000:1.0:1769074928.500432:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954880:192.168.204.121@tcp:101:rm.0 00000100:00100000:1.0:1769074928.501280:0:21385:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.505393:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912954880:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074928.505438:0:21385:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fc0 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.505451:0:21385:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fc0 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.505464:0:21385:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x20:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fc0 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.505476:0:21385:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x20:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fc0 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.505492:0:21385:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff9350062a7800 x1855008912954880/t4294973339(4294973339) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074928.505506:0:21385:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7800 x1855008912954880/t4294973339(4294973339) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.505517:0:21385:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x20:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fc0 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.505528:0:21385:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 16 res 8589935617/32 (0 0) 00000080:00200000:1.0:1769074928.505535:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074928.505560:0:21385:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x20:0x0], updating i_size 120320 00000080:00010000:1.0:1769074928.505566:0:21385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x20:0x0](ffff9350115aba88) for lock 0x410246667097d854 00000080:00010000:1.0:1769074928.505574:0:21385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d854 from it ffffa8e541cdbbd0 00010000:00010000:1.0:1769074928.505592:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x20:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fc0 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.505603:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d854 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x20:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fc0 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.505636:0:21385:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7800 x1855008912954880/t4294973339(4294973339) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074944 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1769074928.505721:0:21385:0:(file.c:6252:ll_getattr_dentry()) START file lockdir:[0x200000401:0x20:0x0](ffff9350115aba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074928.505729:0:21385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88),name=lockdir 00000002:00010000:1.0:1769074928.505755:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x20:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.505767:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.505779:0:21385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.505787:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074928.505793:0:21385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x20:0x0](ffff9350115aba88) 00000080:00010000:1.0:1769074928.505796:0:21385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x20:0x0](ffff9350115aba88) for lock 0x410246667097d846 00000080:00010000:1.0:1769074928.505800:0:21385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d846 from it ffffa8e541cdbcd8 00010000:00010000:1.0:1769074928.505802:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.505809:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074928.505943:0:21385:0:(llite_lib.c:2766:ll_statfs()) VFS Op:sb=lustre (ffff9350067ee000) lockdir proj=0 00000080:00000004:1.0:1769074928.505949:0:21385:0:(obd_class.h:1083:obd_statfs()) lustre-clilmv-ffff9350067ee000: age 332, max_age 638 00800000:00000004:1.0:1769074928.505957:0:21385:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff9350067ee000: age 332, max_age 638 00000100:00100000:1.0:1769074928.506008:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912955008:192.168.204.121@tcp:41:rm.0 00000100:00100000:1.0:1769074928.506140:0:21385:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361ce40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.509458:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912955008:192.168.204.121@tcp:41:rm.0 00800000:00000004:1.0:1769074928.509497:0:21385:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff9350067ee000: update ffff9350076fb520 cache blocks 1882368/1885696 objects 235424/235770 00000080:00000004:1.0:1769074928.509505:0:21385:0:(obd_class.h:1109:obd_statfs()) lustre-clilmv-ffff9350067ee000: update ffff9350076f9c48 cache blocks 1882368/1885696 objects 235424/235770 00000080:00000004:1.0:1769074928.509511:0:21385:0:(llite_lib.c:2629:ll_statfs_internal()) MDC blocks 1882368/1885696 objects 235424/235770 00000080:00200000:1.0:1769074928.509552:0:21385:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88) pos/size0/120320 32bit_api 0 00000080:00200000:1.0:1769074928.509603:0:21385:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88) pos/size18446744073709551614/120320 32bit_api 0 00000080:00200000:1.0:1769074928.509634:0:21385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.509663:0:21385:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile0, dir=[0x200000401:0x20:0x0](ffff9350115aba88), flags=0 00000080:00200000:1.0:1769074928.509668:0:21385:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile0, dir=[0x200000401:0x20:0x0](ffff9350115aba88), intent=getattr 00000002:00010000:1.0:1769074928.509690:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile0,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.509769:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.509780:0:21385:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.509791:0:21385:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00000100:00100000:1.0:1769074928.509814:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912955136:192.168.204.121@tcp:101:rm.0 00000100:00100000:1.0:1769074928.509928:0:21385:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.514521:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912955136:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074928.514553:0:21385:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514574:0:21385:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514593:0:21385:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514607:0:21385:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514620:0:21385:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514640:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514647:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514655:0:21385:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074928.514667:0:21385:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a7480 x1855008912955136/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/640 e 0 to 0 dl 1769074944 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074928.514679:0:21385:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074928.514695:0:21385:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.514705:0:21385:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/30 (0 0) 00000080:00200000:1.0:1769074928.514713:0:21385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x20:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074928.514767:0:21385:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x1e:0x0](ffff9350115ae3c8): rc = 0 00000080:00200000:1.0:1769074928.514782:0:21385:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x1e:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1769074928.514787:0:21385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1e:0x0](ffff9350115ae3c8) for lock 0x410246667097d85b 00000080:00010000:1.0:1769074928.514798:0:21385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1e:0x0](ffff9350115ae3c8) 00000080:00010000:1.0:1769074928.514801:0:21385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d85b from it ffffa8e541cdbdf8 00010000:00010000:1.0:1769074928.514804:0:21385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.514816:0:21385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.514864:0:21385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88), inode mode 41ed mask 3 00000080:00200000:1.0:1769074928.514883:0:21385:0:(namei.c:2354:ll_unlink()) VFS Op:name=lockfile0, dir=[0x200000401:0x20:0x0](ffff9350115aba88) 00010000:00010000:1.0:1769074928.514917:0:21385:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.514937:0:21385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935008f0b800/0x410246667096e53e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x20:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf3edd9 expref: -99 pid: 21356 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.515039:0:21385:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.515079:0:21385:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.515092:0:21385:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935008f0aa00/0x410246667096e4ff lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf3ecc8 expref: -99 pid: 21347 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074928.515100:0:21385:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x1e:0x0]: losing layout lock 00010000:00010000:1.0:1769074928.515131:0:21385:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.515138:0:21385:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935008f0aa00/0x410246667096e4ff lrc: 1/0,0 mode: --/CR res: [0x200000401:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf3ecc8 expref: -99 pid: 21347 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.515144:0:21385:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.515150:0:21385:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1769074928.515153:0:21385:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d85b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf95fe3 expref: -99 pid: 21385 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.515165:0:21385:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935008f0aa00/0x410246667096e4ff lrc: 0/0,0 mode: --/CR res: [0x200000401:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf3ecc8 expref: -99 pid: 21347 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074928.515179:0:21385:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316be00/0x410246667097d846 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x20:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf95fa4 expref: -99 pid: 21385 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074928.515195:0:21385:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074928.515202:0:21385:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7480 x1855008912955392/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074928.515229:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912955392:192.168.204.121@tcp:36:rm.0 00000100:00100000:1.0:1769074928.515323:0:21385:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500361c6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074928.521999:0:21385:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912955392:192.168.204.121@tcp:36:rm.0 00000080:00200000:1.0:1769074928.522154:0:21385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x20:0x0](ffff9350115aba88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074928.522180:0:21385:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=lockfile, dir=[0x200000401:0x20:0x0](ffff9350115aba88), flags=0 00000080:00200000:1.0:1769074928.522185:0:21385:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=lockfile, dir=[0x200000401:0x20:0x0](ffff9350115aba88), intent=getattr 00000002:00010000:1.0:1769074928.522217:0:21385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lockfile,[0x0:0x0:0x0]) in obj [0x200000401:0x20:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074928.522293:0:21385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d862 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.522305:0:21385:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d862 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00010000:00010000:1.0:1769074928.522316:0:21385:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d862 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x20:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21385 initiator: MDT0 00000100:00100000:1.0:1769074928.522342:0:21385:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21385:1855008912955520:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074928.554667:0:14430:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074928.554677:0:14430:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.554686:0:14430:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93500316bc00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074928.554692:0:14430:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.554701:0:14430:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.554723:0:14430:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.554807:0:14430:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.554814:0:14430:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074928.554849:0:14430:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.554857:0:14430:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316bc00/0x410246667096e759 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf3f9fd expref: -99 pid: 21363 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074928.554886:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074928.554888:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074928.554889:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074929.845195:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074930.646905:0:14430:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074930.646915:0:14430:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b400/0x410246667096e73d lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x48bf02eaaaf3f947 expref: -99 pid: 21342 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.646926:0:14430:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93500316b400 already unused, calling callback (ffffffffc137c0e0) 00010000:00010000:1.0:1769074930.646933:0:14430:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b400/0x410246667096e73d lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x48bf02eaaaf3f947 expref: -99 pid: 21342 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.647007:0:14430:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b400/0x410246667096e73d lrc: 2/0,0 mode: --/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x48bf02eaaaf3f947 expref: -99 pid: 21342 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.647017:0:14430:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074930.647045:0:14430:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b400/0x410246667096e73d lrc: 1/0,0 mode: --/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x48bf02eaaaf3f947 expref: -99 pid: 21342 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.647084:0:14430:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b400/0x410246667096e73d lrc: 0/0,0 mode: --/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x48bf02eaaaf3f947 expref: -99 pid: 21342 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.647107:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074930.647109:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074930.647110:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074930.647129:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008912956672:192.168.204.121@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074930.649174:0:14429:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074930.649187:0:14429:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233fc00/0x410246667096e77c lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000010000 nid: local remote: 0x48bf02eaaaf3fa9e expref: -99 pid: 21363 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.649199:0:14429:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93501233fc00 already unused, calling callback (ffffffffc137c0e0) 00010000:00010000:1.0:1769074930.649206:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233fc00/0x410246667096e77c lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400010000 nid: local remote: 0x48bf02eaaaf3fa9e expref: -99 pid: 21363 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074930.649233:0:14429:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x1e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074930.649249:0:14429:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x1e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074930.649312:0:14429:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233fc00/0x410246667096e77c lrc: 2/0,0 mode: --/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400010020 nid: local remote: 0x48bf02eaaaf3fa9e expref: -99 pid: 21363 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.649320:0:14429:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074930.649350:0:14429:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233fc00/0x410246667096e77c lrc: 1/0,0 mode: --/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400010020 nid: local remote: 0x48bf02eaaaf3fa9e expref: -99 pid: 21363 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.649357:0:14429:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233fc00/0x410246667096e77c lrc: 0/0,0 mode: --/PR res: [0x280000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400010020 nid: local remote: 0x48bf02eaaaf3fa9e expref: -99 pid: 21363 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074930.649381:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074930.649383:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074930.649385:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074930.649407:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008912956800:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074930.653311:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008912956800:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074930.653368:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008912956672:192.168.204.121@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074933.938195:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074935.934517:0:21942:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanityn test 13: test directory page revocation ======= 04:42:14 (1769074934) 00000080:00200000:1.0:1769074936.265432:0:21956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074936.265466:0:21956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074936.265485:0:21956:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589934599/1 (0 0) 00010000:00010000:1.0:1769074936.265560:0:21956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21956 initiator: MDT0 00010000:00010000:1.0:1769074936.265571:0:21956:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21956 initiator: MDT0 00010000:00010000:1.0:1769074936.265583:0:21956:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21956 initiator: MDT0 00000100:00100000:1.0:1769074936.265612:0:21956:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:21956:1855008912957824:192.168.204.121@tcp:101:mkdir.0 00000100:00100000:1.0:1769074936.265728:0:21956:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.279615:0:21956:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:21956:1855008912957824:192.168.204.121@tcp:101:mkdir.0 00010000:00010000:1.0:1769074936.279654:0:21956:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279670:0:21956:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279685:0:21956:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279698:0:21956:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279710:0:21956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279719:0:21956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279726:0:21956:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074936.279739:0:21956:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a5c00 x1855008912957824/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074936.279768:0:21956:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279784:0:21956:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074936.279791:0:21956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074936.279828:0:21956:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 12800 00000080:00010000:1.0:1769074936.279836:0:21956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097d869 00000080:00010000:1.0:1769074936.279843:0:21956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074936.279846:0:21956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d869 from it ffffa8e54145bbe0 00010000:00010000:1.0:1769074936.279849:0:21956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279858:0:21956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.279894:0:21956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074936.279931:0:21956:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d13, dir=[0x200000007:0x1:0x0](ffff935011534b08), flags=1536 00000080:00200000:1.0:1769074936.279938:0:21956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074936.279956:0:21956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074936.279966:0:21956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.279983:0:21956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.279996:0:21956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074936.280000:0:21956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074936.280004:0:21956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097d869 00000080:00010000:1.0:1769074936.280007:0:21956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d869 from it ffffa8e54145bce0 00010000:00010000:1.0:1769074936.280010:0:21956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.280017:0:21956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.280025:0:21956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 3 00000080:00200000:1.0:1769074936.280032:0:21956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074936.280038:0:21956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074936.280043:0:21956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.280081:0:21956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.280087:0:21956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074936.280091:0:21956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074936.280093:0:21956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097d869 00000080:00010000:1.0:1769074936.280097:0:21956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d869 from it ffffa8e54145bdc8 00010000:00010000:1.0:1769074936.280099:0:21956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.280105:0:21956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.280113:0:21956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 3 00000080:00200000:1.0:1769074936.280117:0:21956:0:(namei.c:2175:do_mkdir()) VFS Op:name=d13, dir=[0x200000007:0x1:0x0](ffff935011534b08) 00800000:00000004:1.0:1769074936.280140:0:21956:0:(obd_class.h:1049:obd_statfs_async()) lustre-MDT0000-mdc-ffff9350067ee000: age 639, max_age 0 00000100:00100000:1.0:1769074936.280207:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008912957952:192.168.204.121@tcp:41:mkdir.0 00010000:00010000:1.0:1769074936.280357:0:21956:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.280407:0:21956:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.280414:0:21956:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074936.280417:0:21956:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d869 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf9605a expref: -99 pid: 21956 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074936.280440:0:21956:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074936.280447:0:21956:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4a80 x1855008912958208/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 552/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074936.280467:0:21956:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:21956:1855008912958208:192.168.204.121@tcp:36:mkdir.0 00000100:00100000:1.0:1769074936.280493:0:21956:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.297974:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008912957952:192.168.204.121@tcp:41:mkdir.0 00000100:00100000:1.0:1769074936.298044:0:21956:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:21956:1855008912958208:192.168.204.121@tcp:36:mkdir.0 00000080:00200000:1.0:1769074936.298143:0:21956:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248) 00000080:00200000:1.0:1769074936.298153:0:21956:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x21:0x0], updating i_size 512 00000080:00200008:1.0:1769074936.304243:0:21811:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff935011534b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074936.304260:0:21811:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074936.304289:0:21811:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074936.304309:0:21811:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589934599/1 (0 0) 00010000:00010000:1.0:1769074936.304382:0:21811:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21811 initiator: MDT0 00010000:00010000:1.0:1769074936.304394:0:21811:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21811 initiator: MDT0 00010000:00010000:1.0:1769074936.304405:0:21811:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21811 initiator: MDT0 00000100:00100000:1.0:1769074936.304436:0:21811:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job bash:7afd476b-94ee-4c94-b080-d15824f3ae56:21811:1855008912958336:192.168.204.121@tcp:101:bash.0 00000100:00100000:1.0:1769074936.304569:0:21811:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.309942:0:21811:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job bash:7afd476b-94ee-4c94-b080-d15824f3ae56:21811:1855008912958336:192.168.204.121@tcp:101:bash.0 00010000:00010000:1.0:1769074936.309973:0:21811:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.309987:0:21811:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.310003:0:21811:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.310015:0:21811:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.310028:0:21811:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.310037:0:21811:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.310044:0:21811:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074936.310577:0:21811:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a5c00 x1855008912958336/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074936.310596:0:21811:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.310611:0:21811:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074936.310619:0:21811:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074936.326957:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.326982:0:21959:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.326987:0:21959:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.327013:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.327120:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.327131:0:21959:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.327141:0:21959:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000100:00100000:1.0:1769074936.327171:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912958592:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.327283:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.345226:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912958592:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.345255:0:21959:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345272:0:21959:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345280:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345288:0:21959:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345297:0:21959:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345304:0:21959:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.345310:0:21959:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345317:0:21959:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345325:0:21959:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345332:0:21959:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.345340:0:21959:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345347:0:21959:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d87e lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00000002:00100000:1.0:1769074936.345367:0:21959:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912958592/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.345385:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074936.345418:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.345433:0:21959:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074936.345444:0:21959:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d877 from it ffffa8e541cfb980 00010000:00010000:1.0:1769074936.345448:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.345455:0:21959:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.345569:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.345589:0:21959:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.345593:0:21959:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.345614:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.345668:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345675:0:21959:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.345685:0:21959:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000100:00100000:1.0:1769074936.345707:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912958720:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.345970:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.353615:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912958720:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.353645:0:21959:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353672:0:21959:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353679:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353687:0:21959:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353695:0:21959:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353711:0:21959:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.353716:0:21959:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353722:0:21959:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353729:0:21959:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353737:0:21959:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.353753:0:21959:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.353760:0:21959:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d885 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00000002:00100000:1.0:1769074936.353782:0:21959:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912958720/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.353798:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074936.353833:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.353850:0:21959:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074936.353858:0:21959:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d877 from it ffffa8e541cfb980 00010000:00010000:1.0:1769074936.353862:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.353870:0:21959:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.353966:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.353975:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.353987:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.353990:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.353997:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.353999:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.354007:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.354022:0:21959:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.354028:0:21959:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.354045:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.355047:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.355092:0:21959:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.355102:0:21959:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000100:00100000:1.0:1769074936.355136:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912958848:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.355254:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.363635:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912958848:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.363667:0:21959:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363684:0:21959:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363691:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363709:0:21959:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363719:0:21959:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363725:0:21959:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.363732:0:21959:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363739:0:21959:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363758:0:21959:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363765:0:21959:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.363774:0:21959:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.363784:0:21959:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d88c lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00000002:00100000:1.0:1769074936.363803:0:21959:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912958848/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.363821:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074936.363857:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.363872:0:21959:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074936.363881:0:21959:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d877 from it ffffa8e541cfb980 00010000:00010000:1.0:1769074936.363886:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.363894:0:21959:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.363986:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.363997:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.364006:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.364009:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.364019:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.364034:0:21959:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libselinux.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff93501045b380, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074936.364041:0:21959:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libselinux.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074936.364118:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libselinux.so.1,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074936.364180:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.364187:0:21959:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000020:00100000:1.0:1769074936.364199:0:21959:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074936.364205:0:21959:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5180 x1855008912959104/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074936.364214:0:21959:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000100:00100000:1.0:1769074936.364235:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912959104:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.364505:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.370860:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912959104:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.370889:0:21959:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370903:0:21959:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370910:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370918:0:21959:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370927:0:21959:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370934:0:21959:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.370938:0:21959:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370946:0:21959:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370953:0:21959:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370960:0:21959:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.370967:0:21959:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.370973:0:21959:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d893 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00000002:00100000:1.0:1769074936.370995:0:21959:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a5180 x1855008912959104/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.371011:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074936.371045:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.371101:0:21959:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074936.371112:0:21959:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d877 from it ffffa8e541cfb9a8 00010000:00010000:1.0:1769074936.371117:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.371124:0:21959:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.371642:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371655:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1769074936.371668:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371672:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.371682:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371685:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.371695:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371698:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.371708:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371711:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.371720:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371724:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.371733:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371736:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.371759:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371764:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.371774:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.371789:0:21959:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff935004ad7980, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074936.371797:0:21959:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074936.371824:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libcap.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074936.371893:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.371902:0:21959:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000020:00100000:1.0:1769074936.371914:0:21959:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074936.371923:0:21959:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5180 x1855008912959360/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074936.371932:0:21959:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000100:00100000:1.0:1769074936.371958:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912959360:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.372230:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.382269:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912959360:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.382300:0:21959:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382320:0:21959:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382328:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382335:0:21959:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382343:0:21959:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382350:0:21959:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.382355:0:21959:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382361:0:21959:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382368:0:21959:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382375:0:21959:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.382382:0:21959:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.382392:0:21959:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d89a lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00000002:00100000:1.0:1769074936.382418:0:21959:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a5180 x1855008912959360/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.382437:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074936.382488:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.382504:0:21959:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074936.382515:0:21959:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d877 from it ffffa8e541cfb9a8 00010000:00010000:1.0:1769074936.382519:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.382528:0:21959:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.383046:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383127:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1769074936.383143:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383147:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.383157:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383160:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.383169:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383171:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.383180:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383182:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.383191:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383195:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.383204:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383207:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.383215:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383218:0:21959:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.383226:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.383239:0:21959:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff935004ad7c80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074936.383247:0:21959:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074936.383270:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libc.so.6,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074936.383347:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.383356:0:21959:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000020:00100000:1.0:1769074936.383369:0:21959:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074936.383379:0:21959:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5180 x1855008912959616/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074936.383389:0:21959:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000100:00100000:1.0:1769074936.383413:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912959616:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.385146:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.401670:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912959616:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.401967:0:21959:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.401984:0:21959:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.401993:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.402001:0:21959:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.402010:0:21959:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.402017:0:21959:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.402022:0:21959:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.402029:0:21959:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.402036:0:21959:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.402043:0:21959:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.402068:0:21959:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.402075:0:21959:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8a1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00000002:00100000:1.0:1769074936.402100:0:21959:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a5180 x1855008912959616/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.402117:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.453801:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.453816:0:21959:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 44 00000080:00200008:1.0:1769074936.453824:0:21959:0:(file.c:1001:ll_file_open()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), flags 304000 00000002:00010000:1.0:1769074936.453854:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x21:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1769074936.453937:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.453949:0:21959:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000020:00100000:1.0:1769074936.453963:0:21959:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074936.453976:0:21959:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5500 x1855008912960640/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074936.453986:0:21959:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00000100:00100000:1.0:1769074936.454009:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912960640:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.454301:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.457187:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912960640:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.457215:0:21959:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457229:0:21959:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457236:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457243:0:21959:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457251:0:21959:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457262:0:21959:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.457268:0:21959:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457273:0:21959:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457281:0:21959:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457287:0:21959:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.457295:0:21959:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00010000:00010000:1.0:1769074936.457301:0:21959:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8bd lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21959 initiator: MDT0 00000002:00100000:1.0:1769074936.457320:0:21959:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9350062a5500 x1855008912960640/t4294973345(4294973345) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074952 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074936.457335:0:21959:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a5500 x1855008912960640/t4294973345(4294973345) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074952 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.457349:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074936.457374:0:21959:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x21:0x0], updating i_size 512 00000002:00100000:1.0:1769074936.457397:0:21959:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a5500 x1855008912960640/t4294973345(4294973345) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074952 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1769074936.457477:0:21959:0:(file.c:6252:ll_getattr_dentry()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074936.457491:0:21959:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248),name=d13 00000002:00010000:1.0:1769074936.457517:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x21:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074936.457536:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.457551:0:21959:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.457558:0:21959:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074936.457565:0:21959:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x21:0x0](ffff9350115ab248) 00000080:00010000:1.0:1769074936.457568:0:21959:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x21:0x0](ffff9350115ab248) for lock 0x410246667097d877 00000080:00010000:1.0:1769074936.457574:0:21959:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d877 from it ffffa8e541cfbcd8 00010000:00010000:1.0:1769074936.457578:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.457586:0:21959:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.457691:0:21959:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248) pos/size0/512 32bit_api 0 00000002:00010000:1.0:1769074936.458245:0:21959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x21:0x0]) in obj [0x200000401:0x21:0x0], intent: readdir flags 0 00010000:00010000:1.0:1769074936.458258:0:21959:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.458270:0:21959:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074936.459327:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912960768:192.168.204.121@tcp:37:ls.0 00000100:00100000:1.0:1769074936.459469:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.471501:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912960768:192.168.204.121@tcp:37:ls.0 00010000:00010000:1.0:1769074936.472185:0:21959:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.472208:0:21959:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.472281:0:21959:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248) pos/size18446744073709551614/512 32bit_api 0 00000080:00200008:1.0:1769074936.472365:0:21959:0:(file.c:424:ll_file_release()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), flags 304000 00000002:00100000:1.0:1769074936.472431:0:21959:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350062a5500 x1855008912960640/t4294973345(4294973345) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074952 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'ls.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074936.472442:0:21959:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074936.472447:0:21959:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7b80 x1855008912961024/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074936.472472:0:21959:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912961024:192.168.204.121@tcp:35:ls.0 00000100:00100000:1.0:1769074936.472566:0:21959:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.475918:0:21959:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21959:1855008912961024:192.168.204.121@tcp:35:ls.0 00000002:00100000:1.0:1769074936.475955:0:21959:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9350062a5500 x1855008912960640/t4294973345(4294973345) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.479232:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200008:1.0:1769074936.479260:0:21960:0:(file.c:6252:ll_getattr_dentry()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074936.481854:0:21960:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248),name=d13 00000002:00010000:1.0:1769074936.481891:0:21960:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x21:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074936.481914:0:21960:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.481932:0:21960:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.481946:0:21960:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074936.481954:0:21960:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x21:0x0](ffff9350115ab248) 00000080:00010000:1.0:1769074936.481959:0:21960:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x21:0x0](ffff9350115ab248) for lock 0x410246667097d877 00000080:00010000:1.0:1769074936.481966:0:21960:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d877 from it ffffa8e54145bcb8 00010000:00010000:1.0:1769074936.481970:0:21960:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.481979:0:21960:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.486560:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486584:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1769074936.486599:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486603:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.486611:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486614:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.486621:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486623:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.486630:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486632:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074936.486639:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486643:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.486652:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486656:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.486665:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486668:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074936.486676:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.486680:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=libc.so.6, flags=257 00000080:00200000:1.0:1769074936.489943:0:21960:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074936.489978:0:21960:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074936.490002:0:21960:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 12 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.490021:0:21960:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.490030:0:21960:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074936.490038:0:21960:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074936.490042:0:21960:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097d870 00000080:00010000:1.0:1769074936.491804:0:21960:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d870 from it ffffa8e54145bbf8 00010000:00010000:1.0:1769074936.491817:0:21960:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.491830:0:21960:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074936.491850:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074936.491861:0:21960:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d13, flags=25 00000080:00200000:1.0:1769074936.492334:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.492358:0:21960:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 3 00000080:00200000:1.0:1769074936.492362:0:21960:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f13, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7680, open_flags 8941, mode 81b6 opened 0 00000080:00200000:1.0:1769074936.492371:0:21960:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f13, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open|creat 00000002:00010000:1.0:1769074936.492484:0:21960:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f13,[0x200000401:0x22:0x0]) in obj [0x200000401:0x21:0x0], intent: open|creat flags 0104502 00010000:00010000:1.0:1769074936.492505:0:21960:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.492668:0:21960:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.492676:0:21960:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074936.492680:0:21960:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d877 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf96092 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074936.492719:0:21960:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8fc lrc: 3/0,1 mode: --/CW res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21960 initiator: MDT0 00010000:00010000:1.0:1769074936.492725:0:21960:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8fc lrc: 3/0,1 mode: --/CW res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21960 initiator: MDT0 00000020:00100000:1.0:1769074936.492737:0:21960:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074936.492762:0:21960:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7b80 x1855008912961408/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074936.492773:0:21960:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8fc lrc: 3/0,1 mode: --/CW res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21960 initiator: MDT0 00000100:00100000:1.0:1769074936.492798:0:21960:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job touch:7afd476b-94ee-4c94-b080-d15824f3ae56:21960:1855008912961408:192.168.204.121@tcp:101:touch.0 00000100:00100000:1.0:1769074936.492987:0:21960:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.508384:0:21960:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job touch:7afd476b-94ee-4c94-b080-d15824f3ae56:21960:1855008912961408:192.168.204.121@tcp:101:touch.0 00010000:00010000:1.0:1769074936.508427:0:21960:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097d8fc lrc: 4/0,1 mode: --/CW res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9618e expref: -99 pid: 21960 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074936.528159:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008912961920:192.168.204.121@tcp:2:touch.0 00000100:00100000:1.0:1769074936.536014:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008912961920:192.168.204.121@tcp:2:touch.0 00000100:00100000:1.0:1769074936.596360:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963200:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.596391:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596411:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596418:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596425:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596434:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596453:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216e800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.596459:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596467:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596476:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596483:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.596491:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596499:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097d93b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.596532:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350103f8700 x1855008912963200/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.596552:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.596680:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.596701:0:21963:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libselinux.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7380, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074936.596708:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libselinux.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074936.596731:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libselinux.so.1,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074936.596819:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.596827:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000020:00100000:1.0:1769074936.596839:0:21963:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074936.596846:0:21963:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5500 x1855008912963456/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074936.596854:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.596880:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963456:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.597215:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.603655:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963456:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.603684:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603701:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603804:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603812:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603825:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603831:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.603837:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603845:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603853:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603861:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.603868:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.603874:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d942 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.603902:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a5500 x1855008912963456/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.603917:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.604570:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.604592:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.604597:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.604618:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.604691:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.604701:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.605324:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.605355:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963584:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.605538:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.609656:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963584:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.609688:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609714:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609723:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609731:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609748:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609756:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.609761:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609768:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609776:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609784:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.609794:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.609800:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d949 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.609830:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912963584/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.609847:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.609967:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.609985:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.609990:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.610010:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.610102:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.610110:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.610121:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.610143:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963712:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.610233:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.614368:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963712:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.614394:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614408:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614420:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614427:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614436:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614443:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.614448:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614455:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614462:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614470:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.614476:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614483:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d950 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.614509:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912963712/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.614526:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.614632:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.614652:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.614657:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.614678:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.614944:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614953:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.614963:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.614988:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963840:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.615173:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.624410:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963840:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.624524:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624541:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624549:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624556:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624565:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624572:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.624578:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624590:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624598:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624606:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.624614:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624621:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d957 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.624652:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912963840/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.624669:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.624862:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.624879:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.624884:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.624905:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.624966:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624974:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.624985:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.625009:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963968:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.625152:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.629345:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912963968:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.629373:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629387:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629394:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629401:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629410:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629416:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.629422:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629429:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629437:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629457:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.629465:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629472:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d95e lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.629500:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912963968/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.629516:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.629622:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.629641:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.629646:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.629667:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.629744:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629751:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.629762:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.629786:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912964096:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.629883:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.639992:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912964096:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.640023:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640039:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640074:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640083:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640093:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640101:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.640106:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640113:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640120:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640128:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.640136:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.640143:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d965 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.640172:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912964096/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.640189:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.726838:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.726856:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.726861:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.726882:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.726941:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.726948:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.726955:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.726980:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966144:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.727099:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.734176:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966144:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.734206:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734221:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734228:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734236:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734245:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734252:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.734255:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734260:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734266:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734270:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.734277:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734285:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9c7 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.734307:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912966144/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.734321:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.734425:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.734438:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.734442:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.734461:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.734520:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734526:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.734533:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.734550:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966272:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.734633:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.738572:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966272:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.738596:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738607:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738612:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738618:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738625:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738629:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.738633:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738637:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738642:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738647:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.738651:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738655:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ce lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.738675:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912966272/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.738688:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.738814:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.738832:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.738837:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.738857:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.738911:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738917:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.738925:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.738941:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966400:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.739022:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.743283:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966400:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.743311:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743328:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743336:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743343:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743352:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743359:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.743364:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743371:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743383:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743390:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.743399:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743406:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9d5 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.743433:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912966400/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.743461:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.743575:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.743594:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.743598:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.743619:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.743677:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743684:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.743875:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.743902:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966528:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.744218:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.746994:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966528:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.747022:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747036:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747043:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747108:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747116:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747123:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.747127:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747134:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747140:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747149:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.747156:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747162:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9dc lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.747188:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912966528/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.747205:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.747305:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.747321:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.747326:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.747344:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.747398:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747406:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.747416:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.747524:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966656:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.747619:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.752980:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966656:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.753009:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753025:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753032:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753038:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753046:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753153:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.753159:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753249:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753257:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753264:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.753272:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753279:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9e3 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.753308:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912966656/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.753325:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.753434:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.753539:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.753544:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.753563:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.753626:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753633:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.753643:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.753664:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966784:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.754436:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.758905:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966784:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.758933:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.758948:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.758955:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.758961:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.758969:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.758975:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.758980:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.758986:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.758993:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.759000:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.759007:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.759013:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9ea lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.759045:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912966784/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.759107:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.759218:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.759236:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.759240:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.759260:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.759319:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.759326:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.759335:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.759357:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966912:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.763196:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.769886:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912966912:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.769914:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769928:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769935:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769943:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769952:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769958:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.769963:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769970:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769976:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769983:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.769991:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.769996:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097d9f1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.770023:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912966912/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.770040:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.770207:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.770228:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.770233:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.770254:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.770317:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.770324:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.770335:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.770358:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912967040:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.771874:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.776160:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912967040:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.776187:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776202:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776210:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776218:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776225:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776232:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.776236:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776243:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776251:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776258:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.776266:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776273:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097d9f8 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.776300:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5500 x1855008912967040/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.776316:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.776420:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.776449:0:21963:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libpcre2-8.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7800, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074936.776456:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libpcre2-8.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074936.776477:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libpcre2-8.so.0,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074936.776538:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.776546:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000020:00100000:1.0:1769074936.776557:0:21963:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074936.776564:0:21963:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5500 x1855008912967296/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074936.776572:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097d9ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.776592:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912967296:192.168.204.121@tcp:101:ls.0 00000080:00200000:1.0:1769074936.862535:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.862555:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.862560:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.862581:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.862642:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.862650:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.862660:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.862685:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912968960:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.863103:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.868918:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912968960:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.868946:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.868960:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.868968:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.868976:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.868984:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.868990:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.868995:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.869001:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.869010:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.869017:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.869024:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.869029:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da53 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.869076:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912968960/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.869093:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.869198:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.869218:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.869223:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.869241:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.869301:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.869308:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.869319:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.869340:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969088:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.869743:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.874559:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969088:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.874587:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874602:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874609:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874616:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874625:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874631:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.874637:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874644:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874651:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874658:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.874665:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.874671:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097da5a lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.874871:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912969088/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.874889:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.875001:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.875018:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.875023:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.875042:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.875239:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.875247:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.875257:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.875281:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969216:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.875445:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.880008:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969216:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.880037:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880083:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880091:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880099:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880108:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880114:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.880120:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880127:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880135:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880142:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.880150:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880155:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097da61 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.880182:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912969216/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.880204:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.880319:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.880341:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.880346:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.880366:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.880423:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880430:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.880534:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.880561:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969344:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.880905:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.885655:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969344:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.885684:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885902:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885910:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885916:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885924:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885931:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.885936:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885942:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885950:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885957:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.885964:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.885969:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097da68 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.886001:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912969344/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.886018:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074936.886215:0:21963:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074936.886232:0:21963:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074936.886236:0:21963:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074936.886255:0:21963:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074936.886312:0:21963:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.886320:0:21963:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.886331:0:21963:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00000100:00100000:1.0:1769074936.886353:0:21963:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969472:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074936.886575:0:21963:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074936.891234:0:21963:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21963:1855008912969472:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074936.891263:0:21963:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891278:0:21963:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891286:0:21963:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891293:0:21963:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891303:0:21963:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891309:0:21963:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074936.891314:0:21963:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891320:0:21963:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891328:0:21963:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891335:0:21963:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074936.891342:0:21963:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00010000:00010000:1.0:1769074936.891349:0:21963:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097da6f lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21963 initiator: MDT0 00000002:00100000:1.0:1769074936.891377:0:21963:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5180 x1855008912969472/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074952 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074936.891393:0:21963:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000100:00100000:1.0:1769074937.015041:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008912971776:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074937.018790:0:2362:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074937.018821:0:2362:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074937.018846:0:2362:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074937.018859:0:2362:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074937.018876:0:2362:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074937.018888:0:2362:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1769074937.018903:0:2362:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074937.018912:0:2362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074937.018919:0:2362:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074937.018927:0:2362:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216fe00/0x410246667097dac3 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x48bf02eaaaf966ce expref: -99 pid: 21964 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074937.018951:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008912971776:192.168.204.121@tcp:101:rm.0 00000080:00200000:1.0:1769074937.051527:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.051551:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.051556:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.051582:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.051656:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.051669:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.051680:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.051710:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912972288:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.051849:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.065313:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912972288:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.065342:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065359:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065367:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065374:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065382:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065388:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.065393:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065414:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065421:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065428:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.065436:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065442:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097daca lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.065468:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5c00 x1855008912972288/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.065485:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.065610:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.065626:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.065630:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.065650:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.065706:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065714:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.065724:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.065758:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912972416:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.065849:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.077576:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912972416:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.077607:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077625:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077635:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077643:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077651:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077658:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.077664:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077671:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077678:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077686:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.077694:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077701:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dad1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.077721:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5c00 x1855008912972416/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.077749:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.077780:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.077871:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.077891:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.077896:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.077918:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.077982:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.077989:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.078000:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.078027:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912972672:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.079716:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.083016:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912972672:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.083046:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083094:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083101:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083109:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083121:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083127:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.083132:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083139:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083146:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083154:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.083162:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.083169:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097dadf lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.083199:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5c00 x1855008912972672/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.083216:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.083319:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.128132:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.128146:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.128181:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.128260:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097db09 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.128271:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097db09 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.128283:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0400/0x410246667097db09 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.128320:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912973440:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.128429:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd600 going to sleep for 11 seconds 00000080:00200000:1.0:1769074937.166664:0:21966:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.166670:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.166685:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.166716:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.166724:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.166740:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.166755:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912974464:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.166788:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.178738:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912974464:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.178773:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178789:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178796:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178803:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178813:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178819:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.178824:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178831:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178838:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178845:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.178853:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.178859:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db3a lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.178878:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6d80 x1855008912974464/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.178894:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.178923:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.179015:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.179036:0:21966:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libpcre.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff93501045aa80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.179043:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libpcre.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.179125:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libpcre.so.1,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.179195:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.179203:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000020:00100000:1.0:1769074937.179215:0:21966:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 2 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.179221:0:21966:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6d80 x1855008912974720/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.179230:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.179254:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912974720:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.179508:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.185405:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912974720:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.185422:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185433:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185440:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185447:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185453:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185459:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.185464:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185471:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185479:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185486:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.185493:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.185498:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db41 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.185513:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a6d80 x1855008912974720/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.185527:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.185547:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.185892:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.188510:0:21966:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.188519:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.188549:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.188616:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.188625:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.188635:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.189410:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912974976:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.189453:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.199044:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912974976:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.199138:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199154:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199163:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199171:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199182:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199200:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.199208:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199219:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199229:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199258:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.199269:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199279:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097db4f lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.199309:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6d80 x1855008912974976/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.199334:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.199370:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.199473:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.199497:0:21966:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.199502:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.199523:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.199594:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199604:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.199614:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.199636:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912975232:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.199760:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.204575:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912975232:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.204605:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204620:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204627:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204639:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204647:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204654:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.204659:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204666:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204673:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204680:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.204687:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.204693:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097db5d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.204724:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6d80 x1855008912975232/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.204749:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.204878:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00010000:00010000:1.0:1769074937.251400:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.251512:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.251533:0:21965:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff93501045a900, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.251544:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.251566:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libcap.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.251640:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097dba3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.251649:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097dba3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000020:00100000:1.0:1769074937.251663:0:21965:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.251673:0:21965:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6d80 x1855008912976640/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.251684:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097dba3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.251723:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912976640:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.252003:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.256825:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912976256:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.256849:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256863:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256870:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256878:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256885:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256892:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93500316bc00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.256897:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256903:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256910:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256917:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.256924:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.256930:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316bc00/0x410246667097db95 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.256954:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff935002a62680 x1855008912976256/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.256969:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.257087:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.257106:0:21966:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.257110:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.257131:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.257196:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.257204:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.257213:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.257240:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912976768:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.257359:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.265572:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912976768:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.265602:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.265620:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266254:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266350:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266359:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266365:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.266369:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266375:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266384:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266391:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.266398:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266408:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dbaa lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.266435:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912976768/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.266449:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.266475:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.266552:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.266566:0:21966:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.266570:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.266586:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.266629:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266636:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.266713:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.266744:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912977152:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.266837:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.272794:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912977152:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.272818:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272830:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272840:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272846:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272853:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272858:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.272862:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272868:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272874:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272880:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.272885:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.272891:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dbbf lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.272905:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912977152/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.272918:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.272941:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.273005:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.273018:0:21966:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.273022:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.273041:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.273098:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.273105:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.273126:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.273147:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912977408:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.273240:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.281514:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912977408:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.281543:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281560:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281567:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281574:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281583:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281589:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.281595:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281603:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281610:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281618:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.281626:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.281631:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dbcd lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.281652:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912977408/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.281669:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.281700:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.281800:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.281820:0:21966:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff93501045b980, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.281827:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000100:00100000:1.0:1769074937.329601:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912978432:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.329738:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.330101:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:987cd90b-10bf-40f3-bb36-40b816fac7c1:2362:1855008912978560:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074937.336172:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008912978816:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074937.343539:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008912979072:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074937.355567:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912978304:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.355580:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355586:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355592:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355597:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355603:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355607:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93500316b000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.355611:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355616:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355621:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355626:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.355630:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355635:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b000/0x410246667097dbf7 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.355646:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff935002a61880 x1855008912978304/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.355662:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.355674:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.355743:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.355753:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.355757:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.355767:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.355789:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355795:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.355802:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.355814:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912979456:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.355846:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.363167:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912979456:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.363194:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363207:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363213:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363219:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363231:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363237:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.363242:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363248:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363257:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363265:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.363273:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363278:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc0c lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.363295:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912979456/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.363310:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.363337:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.363409:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.363426:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.363432:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.363453:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.363499:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363510:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.363519:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.363540:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912979584:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.363620:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.371540:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912979584:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.371567:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371580:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371587:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371594:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371602:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371608:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.371613:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371620:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371632:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371639:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.371646:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371652:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc13 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.371671:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912979584/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.371697:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.371740:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.371821:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.371841:0:21965:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff93501045b980, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.371849:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.371870:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libc.so.6,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.371927:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.371934:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000020:00100000:1.0:1769074937.371946:0:21965:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.371953:0:21965:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5880 x1855008912979968/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.371960:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.371984:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912979968:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.372226:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.378779:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912979968:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.378795:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378805:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378811:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378819:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378826:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378831:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.378835:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378840:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378846:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378853:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.378857:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.378868:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dc21 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.378892:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a5880 x1855008912979968/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.378903:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.379351:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.379365:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.379371:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.379393:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.379465:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.379475:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.379485:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.379508:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980096:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.379608:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.383357:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980096:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.383377:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383389:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383396:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383404:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383412:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383419:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.383425:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383443:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383451:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383458:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.383466:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383473:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc28 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.383492:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912980096/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.383507:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.383534:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.383615:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.383630:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.383635:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.383653:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.383937:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383946:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.383956:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.383978:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980352:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.384043:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.393509:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980352:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.393538:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393552:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393559:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393566:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393573:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393580:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0c00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.393585:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393591:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393599:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393606:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.393613:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.393620:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc36 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.393641:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912980352/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.393657:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.394478:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.394574:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.394590:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.394595:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.394616:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.395090:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.395101:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.395112:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.395372:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980608:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.395474:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.398533:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912980480:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.398561:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398576:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398583:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398590:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398598:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398603:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935012356400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.398609:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398615:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398621:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398628:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.398635:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.398641:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356400/0x410246667097dc3d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.398661:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff935002a7c380 x1855008912980480/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.398936:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.398966:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.401259:0:21966:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.401293:0:21966:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.401299:0:21966:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.401328:0:21966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.401396:0:21966:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097dc4b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.401407:0:21966:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097dc4b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.401416:0:21966:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097dc4b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00000100:00100000:1.0:1769074937.401530:0:21966:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912980736:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074937.401632:0:21966:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd300 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.403318:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980608:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.403340:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403352:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403360:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403368:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403381:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403387:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.403392:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403398:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403405:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.403413:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.403421:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.404129:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc44 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.404208:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912980608/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.404223:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.404251:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.404339:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.404354:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.404359:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.404379:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.405043:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.405083:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.405094:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.405118:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980864:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.405200:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.412109:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912980864:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.412134:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412147:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412153:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412160:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412167:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412173:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.412179:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412186:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412196:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412202:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.412209:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412215:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc52 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.412233:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912980864/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.412246:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.412270:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.412348:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.412362:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.412367:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.412385:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.412423:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412562:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.412573:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.412595:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912981248:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.412664:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.417519:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912981248:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.417546:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417559:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417565:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417572:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417579:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417585:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.417590:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417598:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417605:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417612:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.417618:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417625:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dc60 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.417643:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912981248/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.417665:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.417701:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.417785:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.417802:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.417807:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.417827:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.417872:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417879:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.417888:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.417909:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912981376:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.417988:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.419833:0:21966:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21966:1855008912981120:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074937.419854:0:21966:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419865:0:21966:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419873:0:21966:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419880:0:21966:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419892:0:21966:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419897:0:21966:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93500316ba00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.419902:0:21966:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419908:0:21966:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419914:0:21966:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419920:0:21966:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.419926:0:21966:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00010000:00010000:1.0:1769074937.419932:0:21966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ba00/0x410246667097dc59 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21966 initiator: MDT0 00000002:00100000:1.0:1769074937.419948:0:21966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff935002a63800 x1855008912981120/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.419962:0:21966:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074937.419984:0:21966:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000100:00100000:1.0:1769074937.424353:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912981376:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.424377:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424390:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424401:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424409:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424417:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424423:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0c00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.424428:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424881:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424889:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424897:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.424905:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.424913:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dc67 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.424943:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912981376/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.424957:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.425092:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.425109:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.425114:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.425135:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.425187:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.425195:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.425205:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.425229:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912981504:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.425391:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.429364:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912981504:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.429395:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429408:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429415:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429421:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429440:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429447:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.429452:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429463:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429470:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429476:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.429482:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.429488:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dc6e lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.429514:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5880 x1855008912981504/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.429527:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.429619:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.429634:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.429638:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.429659:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.430010:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc75 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.430020:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc75 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.430030:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0800/0x410246667097dc75 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.430273:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912981632:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.504818:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693e6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.511673:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912983168:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.511701:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511715:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511732:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511739:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511747:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511754:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216e000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.511759:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511766:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511773:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511781:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.511787:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.511793:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097dcbb lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.512525:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350103f9f80 x1855008912983168/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.512543:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.517082:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.517110:0:21965:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.517598:0:21965:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.517634:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.517707:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.517717:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.517739:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.517768:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912983296:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.517939:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.528469:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912983296:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.528498:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528513:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528521:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528529:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528539:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528546:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.528552:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528559:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528568:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528575:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.528592:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.528599:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dcc2 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.528628:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4000 x1855008912983296/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.528644:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.590672:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.590860:0:21965:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 44 00000080:00200008:1.0:1769074937.590869:0:21965:0:(file.c:1001:ll_file_open()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), flags 304000 00000002:00010000:1.0:1769074937.590903:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x21:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1769074937.590981:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.590993:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000020:00100000:1.0:1769074937.591007:0:21965:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.591024:0:21965:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008912984704/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.591034:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.591703:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912984704:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.591925:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.595893:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912984704:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.595922:0:21965:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.595936:0:21965:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.595943:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.595952:0:21965:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.595960:0:21965:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.595967:0:21965:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.595973:0:21965:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.595981:0:21965:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.595988:0:21965:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.596000:0:21965:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.596007:0:21965:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.596014:0:21965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd01 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21965 initiator: MDT0 00000002:00100000:1.0:1769074937.596035:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9350062a7100 x1855008912984704/t4294973353(4294973353) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074953 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074937.596064:0:21965:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7100 x1855008912984704/t4294973353(4294973353) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074953 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.596078:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074937.596104:0:21965:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x21:0x0], updating i_size 512 00000002:00100000:1.0:1769074937.596125:0:21965:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7100 x1855008912984704/t4294973353(4294973353) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074953 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1769074937.596204:0:21965:0:(file.c:6252:ll_getattr_dentry()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074937.596214:0:21965:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248),name=d13 00000002:00010000:1.0:1769074937.596231:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x21:0x0]) in obj [0x200000401:0x21:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074937.596291:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.596299:0:21965:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00010000:00010000:1.0:1769074937.596310:0:21965:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21965 initiator: MDT0 00000100:00100000:1.0:1769074937.596334:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912984832:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.596415:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.600165:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912984832:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.600209:0:21965:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600238:0:21965:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600250:0:21965:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600262:0:21965:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600287:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600307:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600316:0:21965:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074937.600329:0:21965:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a7480 x1855008912984832/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074937.600341:0:21965:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600354:0:21965:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074937.600363:0:21965:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x21:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074937.600390:0:21965:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x21:0x0], updating i_size 512 00000080:00010000:1.0:1769074937.600396:0:21965:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x21:0x0](ffff9350115ab248) for lock 0x410246667097dd08 00000080:00010000:1.0:1769074937.600402:0:21965:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x21:0x0](ffff9350115ab248) 00000080:00010000:1.0:1769074937.600406:0:21965:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097dd08 from it ffffa8e541c33cd8 00010000:00010000:1.0:1769074937.600410:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600418:0:21965:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074937.600558:0:21965:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248) pos/size0/512 32bit_api 0 00000002:00010000:1.0:1769074937.600590:0:21965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x21:0x0]) in obj [0x200000401:0x21:0x0], intent: readdir flags 0 00010000:00010000:1.0:1769074937.600599:0:21965:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.600625:0:21965:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074937.601573:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912984960:192.168.204.121@tcp:37:ls.0 00000100:00100000:1.0:1769074937.601752:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.622031:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912984960:192.168.204.121@tcp:37:ls.0 00010000:00010000:1.0:1769074937.622792:0:21965:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.622811:0:21965:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097dd08 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96dab expref: -99 pid: 21965 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074937.622879:0:21965:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248) pos/size18446744073709551614/512 32bit_api 0 00000080:00200008:1.0:1769074937.622984:0:21965:0:(file.c:424:ll_file_release()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), flags 304000 00000002:00100000:1.0:1769074937.623102:0:21965:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350062a7100 x1855008912984704/t4294973353(4294973353) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'ls.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074937.623115:0:21965:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074937.623121:0:21965:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7480 x1855008912985216/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074937.623147:0:21965:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912985216:192.168.204.121@tcp:35:ls.0 00000100:00100000:1.0:1769074937.623258:0:21965:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.628622:0:21965:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21965:1855008912985216:192.168.204.121@tcp:35:ls.0 00000002:00100000:1.0:1769074937.628675:0:21965:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9350062a7100 x1855008912984704/t4294973353(4294973353) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'ls.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1769074937.665533:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074937.665541:0:5554:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357600/0x410246667097dd2b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x48bf02eaaaf96e1b expref: -99 pid: 21967 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.665549:0:5554:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935012357600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.665553:0:5554:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357600/0x410246667097dd2b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x11/0x11 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x48bf02eaaaf96e1b expref: -99 pid: 21967 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.665561:0:5554:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357600/0x410246667097dd2b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x11/0x11 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x48bf02eaaaf96e1b expref: -99 pid: 21967 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.665568:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357600/0x410246667097dd2b lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x11/0x11 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf96e1b expref: -99 pid: 21967 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.665598:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357600/0x410246667097dd2b lrc: 2/0,0 mode: --/PR res: [0x200000401:0x21:0x0].0x0 bits 0x11/0x11 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf96e1b expref: -99 pid: 21967 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.665604:0:5554:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074937.665631:0:5554:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357600/0x410246667097dd2b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x21:0x0].0x0 bits 0x11/0x11 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf96e1b expref: -99 pid: 21967 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.665638:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357600/0x410246667097dd2b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x21:0x0].0x0 bits 0x11/0x11 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf96e1b expref: -99 pid: 21967 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074937.665650:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074937.665651:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074937.665653:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074937.665666:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008912986368:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074937.678368:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008912986368:192.168.204.121@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1769074937.751387:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.751409:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.751415:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.751451:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.751518:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.751527:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.751538:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.751568:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912987904:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.751672:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.761494:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912987904:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.761523:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761539:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761547:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761555:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761563:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761570:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.761576:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761583:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761591:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761598:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.761605:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761615:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd63 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.761646:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912987904/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.761664:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.761822:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.761842:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.761846:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.761870:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.761927:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761934:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.761943:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.761973:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988032:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.762085:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.769326:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988032:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.769357:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769371:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769378:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769386:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769395:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769401:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.769407:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769414:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769434:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769442:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.769450:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769458:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd6a lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.769487:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912988032/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.769503:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.769609:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.769626:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.769631:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.769654:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.769739:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769747:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.769757:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.769783:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988160:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.769880:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.773172:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988160:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.773197:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773211:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773219:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773228:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773236:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773243:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.773249:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773255:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773268:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773274:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.773281:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773288:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd71 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.773314:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912988160/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.773329:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.773436:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.773455:0:21968:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libselinux.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e6480, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.773462:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libselinux.so.1, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.773484:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libselinux.so.1,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.773535:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.773543:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000020:00100000:1.0:1769074937.773555:0:21968:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.773562:0:21968:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7480 x1855008912988416/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.773571:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.773593:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988416:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.774291:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.778281:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988416:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.778307:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778322:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778328:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778336:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778344:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778350:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.778355:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778361:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778369:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778377:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.778384:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.778389:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd78 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.778419:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a7480 x1855008912988416/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.778446:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.778941:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.778958:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.778963:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.778983:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.779036:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.779044:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.779090:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.779115:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988544:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.779211:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.785974:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988544:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.786000:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786014:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786021:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786029:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786038:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786044:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.786067:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786075:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786083:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786089:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.786096:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786102:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd7f lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.786130:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912988544/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.786146:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.786246:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.786262:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.786268:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.786291:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.786343:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786350:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.786360:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.786381:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988672:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.786480:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.790415:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988672:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.790451:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790465:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790471:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790479:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790488:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790494:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.790498:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790505:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790511:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790517:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.790524:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790530:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dd86 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.790558:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912988672/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.790573:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.790667:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.790688:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.790693:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.790735:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.790787:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790793:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.790802:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.790825:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988800:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.790913:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.794617:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988800:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.794644:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794658:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794669:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794689:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794697:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794828:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.794835:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794843:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794850:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794857:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.794863:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.794870:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dd8d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.794897:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912988800/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.794913:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.795008:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.795023:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.795028:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.795072:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.795120:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.795128:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.795137:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.795159:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988928:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.795295:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.801209:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912988928:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.801238:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801251:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801258:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801264:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801271:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801277:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.801282:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801293:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801300:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801307:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.801313:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801319:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097dd94 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.801344:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912988928/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.801359:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.801463:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.801479:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.801484:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.801504:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.801555:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801563:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.801573:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.801598:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989056:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.801692:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.806611:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989056:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.806638:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806651:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806659:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806667:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806676:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806682:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.806688:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806695:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806714:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806732:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.806740:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806746:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dd9b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.806774:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912989056/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.806791:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.806888:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.806907:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.806912:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.806931:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.806982:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.806990:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.807000:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.807022:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989184:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.807161:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.811090:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989184:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.811116:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811129:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811136:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811144:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811152:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811159:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.811164:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811171:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811178:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811186:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.811193:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811200:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dda2 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.811225:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912989184/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.811241:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.811327:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.811341:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.811346:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.811367:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.811413:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811432:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.811442:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.811463:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989312:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.811548:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.818093:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989312:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.818121:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818135:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818141:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818148:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818156:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818163:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.818168:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818174:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818180:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818187:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.818194:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818203:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dda9 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.818232:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912989312/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.818248:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.818349:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.818368:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.818374:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.818395:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.818459:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818466:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.818475:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.818498:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989440:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.818587:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.822404:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989440:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.822512:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822526:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822534:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822541:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822548:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822555:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.822560:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822568:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822575:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822582:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.822590:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.822596:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097ddb0 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.822626:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912989440/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.822641:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.823432:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.823453:0:21968:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e6d80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.823462:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.823489:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libcap.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.823549:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.823559:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000020:00100000:1.0:1769074937.823571:0:21968:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.823579:0:21968:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7480 x1855008912989696/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.823588:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.823612:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989696:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.823827:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.827109:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989696:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.827136:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827149:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827155:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827162:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827170:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827176:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.827182:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827189:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827196:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827202:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.827209:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827216:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097ddb7 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.827243:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a7480 x1855008912989696/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.827259:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.827660:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.827675:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.827680:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.827704:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.827768:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827777:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.827786:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.827809:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989824:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.827906:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.831491:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989824:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.831516:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831528:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831535:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831542:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831550:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831556:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.831562:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831569:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831577:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831584:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.831591:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831597:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097ddbe lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.831632:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912989824/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.831646:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.831784:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.831799:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.831804:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.831826:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.831876:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831884:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.831894:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.831918:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989952:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.832004:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.835008:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912989952:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.835038:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835303:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835311:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835319:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835327:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835333:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.835338:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835345:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835352:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835359:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.835366:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.835373:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097ddc5 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.835402:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7480 x1855008912989952/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.835417:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000100:00100000:1.0:1769074937.878198:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991232:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.878221:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878233:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878240:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878249:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878256:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878263:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216e800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.878268:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878274:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878282:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878289:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.878296:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878302:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097de04 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.878330:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350103fb800 x1855008912991232/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.878345:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.878524:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.878542:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.878547:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.878570:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.878624:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878631:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.878641:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.878664:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991360:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.879297:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.883082:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991360:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.883107:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883121:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883128:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883136:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883145:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883151:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.883156:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883163:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883174:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883181:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.883188:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883195:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de0b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.883221:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912991360/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.883235:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.883322:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.883339:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.883343:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.883366:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.883413:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883432:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.883441:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.883465:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991488:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.883552:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.887924:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991488:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.887953:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.887970:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.887976:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.887984:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.887991:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.887998:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.888003:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.888010:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.888017:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.888024:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.888030:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.888037:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de12 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.888096:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912991488/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.888111:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.888205:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.888220:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.888225:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.888245:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.888292:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.888299:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.888309:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.888329:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991616:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.888414:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.892364:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991616:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.892393:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892406:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892413:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892433:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892442:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892447:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.892453:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892460:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892467:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892474:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.892481:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892487:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de19 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.892515:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912991616/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.892530:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.892626:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.892641:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.892647:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.892668:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.892934:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892942:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.892951:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.892973:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991744:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.893250:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.897590:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991744:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.897622:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897638:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897645:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897653:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897663:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897850:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.897857:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897865:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897872:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897879:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.897886:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.897892:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de20 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.897922:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912991744/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.897939:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.898029:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.898081:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.898088:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.898111:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.898167:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.898175:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.898184:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.898207:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991872:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.898363:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.902891:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912991872:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.902914:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902929:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902935:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902943:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902952:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902959:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.902964:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902971:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902978:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902985:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.902991:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.902998:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de27 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.903024:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912991872/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.903038:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.903167:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.903183:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.903188:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.903209:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.903260:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.903268:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.903278:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.903302:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992000:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.903389:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.907734:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992000:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.907766:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907780:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907787:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907794:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907801:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907807:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.907812:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907818:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907826:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907833:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.907842:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.907847:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de2e lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.907873:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912992000/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.907887:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.907983:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.908000:0:21968:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libpcre2-8.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e6d80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.908008:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libpcre2-8.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.908031:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libpcre2-8.so.0,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.908129:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.908136:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000020:00100000:1.0:1769074937.908149:0:21968:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.908155:0:21968:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008912992256/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.908164:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.908189:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992256:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.908369:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.913041:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992256:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.913097:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913112:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913119:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913127:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913134:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913141:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.913145:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913153:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913160:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913167:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.913174:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.913180:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de35 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.913209:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a7100 x1855008912992256/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.913225:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.913932:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.913954:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.913960:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.913983:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.914041:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.914089:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.914101:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.914127:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992384:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.914231:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.917519:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992384:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.917546:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917559:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917565:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917572:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917579:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917585:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.917590:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917597:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917604:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917610:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.917616:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.917622:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de3c lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.917650:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912992384/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.917665:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.917995:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.918011:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.918015:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.918034:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.918111:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.918118:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.918129:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.918152:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992512:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.918283:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.921490:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992512:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.921515:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921529:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921536:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921544:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921552:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921558:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.921562:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921569:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921576:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921584:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.921590:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921596:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097de43 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.921629:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912992512/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.921644:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.921772:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.921791:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.921796:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.921817:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.921863:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921871:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.921881:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.921904:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992640:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.922700:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.925657:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992640:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.926020:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926035:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926042:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926244:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926256:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926262:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.926267:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926274:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926281:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926287:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.926295:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926301:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097de4a lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.926333:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912992640/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.926351:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.926453:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.926468:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.926473:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.926495:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.926541:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926548:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.926559:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.926581:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992768:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.926679:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.929767:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992768:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.929792:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929805:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929812:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929820:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929828:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929835:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.929840:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929847:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929854:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929861:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.929869:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.929875:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de51 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.929904:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912992768/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.929919:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.930006:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.930021:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.930026:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.930045:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.930131:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.930138:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.930147:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.930167:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992896:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.930241:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.933216:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912992896:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.933240:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933252:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933263:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933270:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933277:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933282:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.933286:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933292:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933298:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933304:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.933311:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933317:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de58 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.933342:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912992896/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.933357:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.933456:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.933471:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.933476:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.933495:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.933541:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933549:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.933558:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.933581:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993024:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.933651:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.936907:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993024:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.936929:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936941:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936948:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936956:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936963:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936968:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.936973:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936985:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936992:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.936998:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.937003:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.937009:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de5f lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.937032:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912993024/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.937046:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.937148:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.937164:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.937169:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.937189:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.937237:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.937244:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.937254:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.937275:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993152:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.937351:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.940893:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993152:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.940924:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940938:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940945:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940951:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940958:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940964:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.940968:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940975:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940981:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.940989:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.940995:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.941000:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de66 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.941026:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912993152/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.941042:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.941168:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.941184:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.941189:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.941209:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.941255:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.941262:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.941271:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.941292:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993280:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.941376:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.945174:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993280:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.945200:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945214:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945221:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945229:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945238:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945246:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.945251:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945258:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945266:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945273:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.945280:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945287:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de6d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.945315:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912993280/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.945329:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.945441:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.945460:0:21968:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libdl.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e6f00, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.945468:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libdl.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.945490:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libdl.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.945546:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.945555:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000020:00100000:1.0:1769074937.945568:0:21968:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.945575:0:21968:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008912993536/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.945584:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.945606:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993536:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.945800:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.949515:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993536:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.949541:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949555:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949562:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949569:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949577:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949584:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.949588:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949596:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949604:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949612:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.949618:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.949623:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de74 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.949649:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a7100 x1855008912993536/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.949665:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.950112:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.950128:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.950133:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.950155:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.950204:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.950212:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.950222:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.950244:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993664:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.950332:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.953942:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993664:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.953968:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.953986:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.953993:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954000:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954008:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954015:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.954019:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954027:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954034:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954040:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.954119:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954128:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de7b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.954152:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912993664/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.954166:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.954260:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.954278:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.954284:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.954305:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.954356:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954364:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.954374:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.954398:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993792:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.954864:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.958016:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993792:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.958040:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958122:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958129:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958137:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958145:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958151:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.958157:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958163:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958171:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958178:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.958186:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958192:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de82 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.958219:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912993792/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.958235:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.958324:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.958338:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.958343:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.958361:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.958406:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958413:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.958496:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.958518:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993920:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.958650:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.966924:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912993920:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.966953:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.966968:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.966975:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.966982:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.966989:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.966995:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.967000:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.967006:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.967012:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.967020:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.967026:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.967031:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097de89 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.967178:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912993920/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.967194:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.967289:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.967304:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.967308:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.967331:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.967384:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.967392:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.967402:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.967507:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994048:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.967649:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.971574:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994048:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.971599:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971613:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971620:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971627:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971636:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971642:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.971647:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971653:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971660:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971667:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.971828:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.971836:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097de90 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.971864:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912994048/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.971879:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.971966:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.971980:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.971986:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.972007:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.972131:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.972139:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.972151:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.972175:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994176:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.972306:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.975761:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994176:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.975785:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975797:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975805:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975812:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975820:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975826:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.975831:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975838:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975845:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975851:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.975857:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.975863:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097de97 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.975889:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912994176/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.975904:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.975990:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.976005:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.976009:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.976028:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.976109:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.976116:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.976125:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.976147:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994304:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.976232:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.979864:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994304:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.979890:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979903:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979914:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979921:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979929:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979935:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.979940:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979947:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979954:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979961:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.979969:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.979975:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097de9e lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.980004:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912994304/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.980018:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.980137:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.980151:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.980156:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.980174:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.980221:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.980228:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.980237:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.980260:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994432:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.980349:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.983796:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994432:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.983820:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983833:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983839:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983846:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983854:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983861:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.983866:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983877:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983884:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983891:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.983898:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.983905:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dea5 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.983932:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912994432/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.983947:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.984034:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.984392:0:21968:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074937.984399:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074937.984544:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074937.984600:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.984608:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.984618:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.984642:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994560:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.984954:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.988393:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994560:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.988500:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988515:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988522:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988530:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988538:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988544:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.988549:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988556:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988563:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988570:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.988578:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.988585:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097deac lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.988614:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912994560/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.988629:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074937.989080:0:21968:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074937.989101:0:21968:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libpthread.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7c80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074937.989108:0:21968:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libpthread.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074937.989131:0:21968:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libpthread.so.0,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074937.989192:0:21968:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.989201:0:21968:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000020:00100000:1.0:1769074937.989213:0:21968:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074937.989221:0:21968:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008912994816/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074937.989231:0:21968:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00000100:00100000:1.0:1769074937.989255:0:21968:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994816:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074937.989591:0:21968:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074937.993344:0:21968:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21968:1855008912994816:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074937.993370:0:21968:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993383:0:21968:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993390:0:21968:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993402:0:21968:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993409:0:21968:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993415:0:21968:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074937.993508:0:21968:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993516:0:21968:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993523:0:21968:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993529:0:21968:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074937.993537:0:21968:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00010000:00010000:1.0:1769074937.993543:0:21968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097deb3 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21968 initiator: MDT0 00000002:00100000:1.0:1769074937.993572:0:21968:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a7100 x1855008912994816/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074953 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074937.993587:0:21968:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000100:00100000:1.0:1769074938.049852:0:21969:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21969:1855008912995712:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074938.049880:0:21969:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049893:0:21969:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049901:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049908:0:21969:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049917:0:21969:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049923:0:21969:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216e600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.049928:0:21969:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049935:0:21969:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049942:0:21969:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049949:0:21969:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.049955:0:21969:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.049961:0:21969:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dec8 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00000002:00100000:1.0:1769074938.049980:0:21969:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350103f8700 x1855008912995712/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.049995:0:21969:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.050024:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.050041:0:21969:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074938.050128:0:21969:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097dec1 from it ffffa8e541ca3980 00010000:00010000:1.0:1769074938.050133:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.050143:0:21969:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.050222:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.050237:0:21969:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.050242:0:21969:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.050264:0:21969:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.050370:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.050378:0:21969:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.050388:0:21969:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00000100:00100000:1.0:1769074938.050412:0:21969:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21969:1855008912995840:192.168.204.121@tcp:101:rm.0 00000100:00100000:1.0:1769074938.050959:0:21969:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.055314:0:21969:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21969:1855008912995840:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074938.055342:0:21969:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055361:0:21969:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055368:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055376:0:21969:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055384:0:21969:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055391:0:21969:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.055396:0:21969:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055402:0:21969:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055409:0:21969:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055417:0:21969:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.055423:0:21969:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.055430:0:21969:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097decf lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00000002:00100000:1.0:1769074938.055448:0:21969:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912995840/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.055463:0:21969:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.055492:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.055504:0:21969:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074938.055513:0:21969:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097dec1 from it ffffa8e541ca3980 00010000:00010000:1.0:1769074938.055517:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.055535:0:21969:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.055609:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.055619:0:21969:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074938.055631:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.055635:0:21969:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074938.055643:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.055646:0:21969:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1769074938.055655:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.055665:0:21969:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.055680:0:21969:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.055700:0:21969:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.062243:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.062257:0:21969:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.062269:0:21969:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00000100:00100000:1.0:1769074938.062317:0:21969:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21969:1855008912995968:192.168.204.121@tcp:101:rm.0 00000100:00100000:1.0:1769074938.062431:0:21969:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.069024:0:21969:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21969:1855008912995968:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074938.069067:0:21969:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069079:0:21969:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069085:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069092:0:21969:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069098:0:21969:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069104:0:21969:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.069109:0:21969:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069116:0:21969:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069124:0:21969:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069130:0:21969:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.069138:0:21969:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.069154:0:21969:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097ded6 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00000002:00100000:1.0:1769074938.069173:0:21969:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7100 x1855008912995968/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.069188:0:21969:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.069216:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.069235:0:21969:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074938.069250:0:21969:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097dec1 from it ffffa8e541ca3980 00010000:00010000:1.0:1769074938.069257:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.069268:0:21969:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.070556:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.070569:0:21969:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074938.070583:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.070586:0:21969:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1769074938.070595:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.070612:0:21969:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e6000, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074938.070620:0:21969:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074938.070648:0:21969:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libc.so.6,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074938.070727:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.070736:0:21969:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00000020:00100000:1.0:1769074938.070748:0:21969:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074938.070757:0:21969:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008912996224/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074938.070766:0:21969:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00000100:00100000:1.0:1769074938.070791:0:21969:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21969:1855008912996224:192.168.204.121@tcp:101:rm.0 00000100:00100000:1.0:1769074938.070991:0:21969:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.079688:0:21969:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:7afd476b-94ee-4c94-b080-d15824f3ae56:21969:1855008912996224:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074938.079759:0:21969:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079774:0:21969:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079780:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079788:0:21969:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079796:0:21969:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079802:0:21969:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.079807:0:21969:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079815:0:21969:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079822:0:21969:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079830:0:21969:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.079837:0:21969:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.079843:0:21969:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dedd lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21969 initiator: MDT0 00000002:00100000:1.0:1769074938.079863:0:21969:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a7100 x1855008912996224/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.079879:0:21969:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.079912:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.079924:0:21969:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074938.079933:0:21969:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097dec1 from it ffffa8e541ca39a8 00010000:00010000:1.0:1769074938.079937:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.079944:0:21969:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.081098:0:21969:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074938.081128:0:21969:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074938.081144:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.081157:0:21969:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.081165:0:21969:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074938.081172:0:21969:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074938.081177:0:21969:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667097daae 00000080:00010000:1.0:1769074938.081184:0:21969:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097daae from it ffffa8e541ca3b58 00010000:00010000:1.0:1769074938.081187:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.081196:0:21969:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.081210:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074938.081219:0:21969:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d13, flags=24 00000080:00200000:1.0:1769074938.081227:0:21969:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115f5348), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.081239:0:21969:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f13, dir=[0x200000401:0x21:0x0](ffff9350115f5348), flags=8 00000080:00200000:1.0:1769074938.081244:0:21969:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f13, dir=[0x200000401:0x21:0x0](ffff9350115f5348), intent=getattr 00000002:00010000:1.0:1769074938.081259:0:21969:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f13,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074938.081308:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.081316:0:21969:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00010000:00010000:1.0:1769074938.081325:0:21969:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21969 initiator: MDT0 00000100:00100000:1.0:1769074938.081353:0:21969:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:84af2899-c956-435d-9e3c-5d5fbc3c9246:21969:1855008912996352:192.168.204.121@tcp:101:rm.0 00000100:00100000:1.0:1769074938.081462:0:21969:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.087404:0:21969:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job rm:84af2899-c956-435d-9e3c-5d5fbc3c9246:21969:1855008912996352:192.168.204.121@tcp:101:rm.0 00010000:00010000:1.0:1769074938.088241:0:21969:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.088271:0:21969:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x23:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.091342:0:21969:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.091361:0:21969:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.091376:0:21969:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.091398:0:21969:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.091405:0:21969:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.091412:0:21969:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074938.091430:0:21969:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a7100 x1855008912996352/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 576/640 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074938.091444:0:21969:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0000/0x410246667097dee4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97346 expref: -99 pid: 21969 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.113521:0:14430:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074938.113540:0:14430:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.113555:0:14430:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216f800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.113562:0:14430:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.113571:0:14430:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.113580:0:14430:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097dec1 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x21:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x48bf02eaaaf972d6 expref: -99 pid: 21968 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074938.207646:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fa680 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008912998528:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.207657:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207663:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207669:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207675:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207686:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207691:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216e600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.207695:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207700:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207707:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207725:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.207730:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207736:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097df46 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.207751:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350103fa680 x1855008912998528/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.207763:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.207777:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.207828:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.207842:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.207846:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.207859:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.207895:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207903:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.207908:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.207919:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008912998912:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.217193:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.230454:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008912998912:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.230469:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230477:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230483:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230491:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230497:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230502:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.230506:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230512:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230523:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230529:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.230533:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230539:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097df54 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.230556:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5f80 x1855008912998912/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.230566:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.230605:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.230612:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.230616:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.230623:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.230638:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230644:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.230649:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.230659:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008912999168:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.230770:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.237484:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008912999168:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.237505:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237514:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237520:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237528:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237535:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237541:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.237546:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237552:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237560:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237567:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.237573:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.237579:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df62 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.237597:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a5f80 x1855008912999168/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.237617:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000100:00100000:1.0:1769074938.276430:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8000 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913000064:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.276458:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276472:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276479:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276486:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276493:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276500:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216f800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.276504:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276511:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276517:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276524:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.276531:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.276537:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216f800/0x410246667097df8c lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.276560:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350103f8000 x1855008913000064/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.276574:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.276601:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.277020:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.277081:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.277087:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.277109:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.277157:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.277166:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.277175:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.277198:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913000192:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.277346:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.280777:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913000192:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.280804:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280817:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280824:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280836:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280844:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280850:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.280855:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280861:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280868:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280874:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.280880:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.280886:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097df93 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.280904:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7800 x1855008913000192/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.280918:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.280945:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.281021:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.281480:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.281486:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.281509:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.281563:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.281572:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.281582:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.281605:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913000448:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.281692:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.314996:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913000448:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.315007:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315014:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315020:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315026:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315043:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315088:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.315092:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315098:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315105:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315111:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.315115:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315120:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfa1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.315131:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7800 x1855008913000448/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.315141:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.315151:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.315190:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.315200:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.315203:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.315211:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.315243:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315250:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.315258:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.315270:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913000832:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.315296:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.331372:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913000832:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.331397:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331411:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331424:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331431:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331438:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331444:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.331449:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331456:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331463:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331469:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.331477:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331483:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfb6 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.331500:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7800 x1855008913000832/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.331517:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.331540:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.331602:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.331613:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.331617:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.331643:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.331686:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331693:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.331703:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.331737:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913001216:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.331829:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.343041:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913001216:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.343101:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343117:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343125:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343134:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343148:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343154:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.343160:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343168:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343176:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343182:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.343191:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343197:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfc4 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.343215:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7800 x1855008913001216/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.343245:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.343271:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.343355:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.343370:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.343374:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.343396:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.343458:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343469:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.343477:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.343500:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913001344:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.343598:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.360793:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913001344:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.360805:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360813:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360818:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360823:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360826:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360829:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.360831:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360834:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360840:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360844:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.360846:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.360849:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097dfcb lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.360862:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7800 x1855008913001344/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.360870:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.361632:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.361642:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.361647:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.361657:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.361681:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.361690:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.361696:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.361847:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913001728:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.361935:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.364845:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913001728:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.364872:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364885:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364893:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364900:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364907:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364913:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.364918:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364924:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364931:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364939:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.364946:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.364952:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097dfe0 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.364970:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a7800 x1855008913001728/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.364986:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.365010:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.365121:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.365137:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.365142:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.365159:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.365203:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dfe7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.365211:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dfe7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.365219:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097dfe7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.365254:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913001856:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.365332:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cd480 going to sleep for 11 seconds 00010000:00010000:1.0:1769074938.366430:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.366477:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.366485:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.366488:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.366498:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.366521:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.366526:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.366532:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.366544:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913001984:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.366603:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.373094:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913001984:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.373111:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373122:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373129:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373137:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373143:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373149:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.373152:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373158:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373166:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373173:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.373178:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.373188:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097dfee lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.373205:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913001984/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.373217:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.373234:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000100:00100000:1.0:1769074938.377693:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8a80 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913002240:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.377893:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377904:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377910:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377916:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377923:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377928:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216e600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.377931:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377937:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377944:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377952:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.377959:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.377965:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e600/0x410246667097dffc lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.377986:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350103f8a80 x1855008913002240/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.378001:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.378169:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.378245:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.378261:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.378266:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.378286:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.378332:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.378340:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.378351:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.378374:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913002368:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.378554:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.383140:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913002368:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.383166:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383184:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383190:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383197:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383205:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383211:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.383216:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383223:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383230:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383237:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.383245:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383252:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e003 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.383270:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913002368/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.383287:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.383316:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.383391:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.383408:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.383413:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.383434:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.383481:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383486:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.383494:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.383511:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913002752:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.383592:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.387836:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913002752:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.387848:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387857:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387864:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387872:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387881:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387892:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.387897:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387912:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387920:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387928:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.387934:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.387941:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e011 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.387959:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913002752/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.387974:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.388230:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.388247:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.388252:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.388272:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.388324:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.388331:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.388341:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.388365:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913002880:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.388456:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.392620:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913002880:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.392644:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392656:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392674:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392682:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392689:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392695:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.392701:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392717:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392724:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392731:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.392736:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392755:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e018 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.392773:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913002880/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.392787:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.392813:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.392880:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.392906:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.392911:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.392929:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.392969:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392976:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.392986:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.393016:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913003136:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.393118:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.407144:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913003136:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.407188:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407201:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407212:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407220:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407228:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407233:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.407238:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407244:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407251:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407258:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.407265:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407272:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e026 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.407291:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913003136/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.407306:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.407336:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.407433:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.407452:0:21970:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7380, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074938.407460:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libc.so.6, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074938.407483:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libc.so.6,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074938.407541:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.407548:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000020:00100000:1.0:1769074938.407560:0:21970:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074938.407566:0:21970:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4e00 x1855008913003520/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074938.407574:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.407596:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913003520:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.409020:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.412369:0:21971:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913003264:192.168.204.121@tcp:101:grep.0 00010000:00010000:1.0:1769074938.412404:0:21971:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412421:0:21971:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412429:0:21971:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412437:0:21971:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412450:0:21971:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412456:0:21971:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216e000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.412462:0:21971:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412468:0:21971:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412477:0:21971:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412484:0:21971:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.412492:0:21971:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412498:0:21971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e02d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21971 initiator: MDT0 00000002:00100000:1.0:1769074938.412522:0:21971:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350103fb480 x1855008913003264/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.412538:0:21971:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.412571:0:21971:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.412666:0:21971:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.412684:0:21971:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.412689:0:21971:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.412721:0:21971:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.412783:0:21971:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e03b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412794:0:21971:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e03b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00010000:00010000:1.0:1769074938.412805:0:21971:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1400/0x410246667097e03b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21971 initiator: MDT0 00000100:00100000:1.0:1769074938.412828:0:21971:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job grep:7afd476b-94ee-4c94-b080-d15824f3ae56:21971:1855008913003648:192.168.204.121@tcp:101:grep.0 00000100:00100000:1.0:1769074938.413514:0:21971:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.419363:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913003520:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.419404:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419421:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419429:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419437:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419446:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419452:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.419458:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419463:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419473:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419478:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.419484:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.419489:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e034 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.419510:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a4e00 x1855008913003520/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.419525:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.419556:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.420032:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.420047:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.420103:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.420124:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.422488:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.422761:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.422783:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.422832:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913003776:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.423153:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.432565:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913003776:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.432585:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432592:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432597:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432603:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432609:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432614:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.432618:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432622:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432627:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432632:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.432636:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.432641:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e042 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.432659:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913003776/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.432675:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.432731:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.437234:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.437242:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.437268:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.437331:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.437340:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.437350:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.437375:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004160:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.438230:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.444543:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004160:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.444574:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444587:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444594:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444600:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444607:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444614:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.444618:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444626:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444632:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444640:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.444648:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444653:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e057 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.444671:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913004160/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.444683:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.444714:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.444795:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.444821:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.444825:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.444842:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.444901:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444909:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.444921:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.444941:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004544:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.445013:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.451286:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004544:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.451316:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451332:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451339:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451347:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451355:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451361:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.451367:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451373:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451381:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451389:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.451398:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451405:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e06c lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.451438:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913004544/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.451456:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.451487:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.451766:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.451790:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.451795:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.451816:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.451968:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451976:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.451987:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.452013:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004672:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.452161:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.457846:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004672:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.457886:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457900:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457908:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457915:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457922:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457928:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.457933:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457940:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457946:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457954:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.457962:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.457968:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e073 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.457989:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913004672/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.458014:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.458047:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.458153:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.458171:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.458176:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.458196:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.458248:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.458256:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.458265:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.458288:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004800:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.458373:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.467490:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913004800:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.467520:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467534:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467541:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467548:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467557:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467564:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.467743:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467752:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467760:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467767:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.467775:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.467781:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e07a lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.467804:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913004800/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.467820:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074938.467850:0:21970:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/33 (0 0) 00000080:00200000:1.0:1769074938.468422:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.468443:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.468448:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.468472:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.468543:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.468553:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.468563:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.468697:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005312:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.468910:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.479837:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005312:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.479879:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479893:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479900:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479907:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479914:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479921:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.479926:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479933:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479940:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479948:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.479956:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.479962:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e08f lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.479995:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913005312/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.480021:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.480147:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.480164:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.480168:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.480188:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.480247:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.480266:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.480277:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.480301:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005440:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.480412:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.484074:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005440:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.484106:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484121:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484128:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484136:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484145:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484151:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.484157:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484165:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484172:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484179:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.484185:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484191:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e096 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.484218:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913005440/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.484234:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.484337:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.484352:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.484356:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.484375:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.484428:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484433:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.484441:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.484459:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005568:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.484541:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.488921:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005568:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.488950:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.488963:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.488969:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.488976:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.488983:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.488989:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.488994:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.489001:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.489008:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.489016:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.489023:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.489030:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e09d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.489092:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a4e00 x1855008913005568/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.489109:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.489204:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.489220:0:21970:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libpcre2-8.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7200, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074938.489228:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libpcre2-8.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074938.489251:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libpcre2-8.so.0,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074938.489307:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.489316:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000020:00100000:1.0:1769074938.489328:0:21970:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074938.489335:0:21970:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4e00 x1855008913005824/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074938.489343:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.489365:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005824:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.489556:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.493153:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005824:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.493185:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493202:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493209:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493216:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493225:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493231:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.493235:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493243:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493251:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493259:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.493265:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.493272:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0a4 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.493302:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a4e00 x1855008913005824/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.493317:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.493906:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.493923:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.493928:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.493951:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.494007:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e0ab lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.494015:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e0ab lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.494025:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e0ab lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.494099:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913005952:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.520031:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.520086:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.520099:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.520134:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913006592:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.520308:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.524366:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913006592:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.524391:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524404:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524496:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524505:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524512:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524518:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.524523:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524530:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524537:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524544:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.524550:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.524557:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ce lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.524587:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350103fb480 x1855008913006592/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.524603:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.524895:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.524913:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.524919:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.524939:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.524997:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.525004:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.525013:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.525036:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913006720:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.525214:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.533242:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913006848:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.533267:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533281:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533287:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533294:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533301:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533307:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93500316b600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.533312:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533323:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533330:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533337:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.533344:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533350:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e0dc lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.533377:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff935002a62680 x1855008913006848/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.533392:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.533495:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.533512:0:21970:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libdl.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7680, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074938.533519:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libdl.so.2, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074938.533542:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libdl.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074938.533595:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.533603:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000020:00100000:1.0:1769074938.533615:0:21970:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074938.533621:0:21970:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6680 x1855008913007104/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074938.533630:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.533658:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007104:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.534459:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.538811:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007104:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.538836:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538847:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538853:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538860:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538867:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538873:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.538878:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538885:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538893:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538899:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.538906:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.538916:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0e3 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.538940:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a6680 x1855008913007104/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.538953:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.539374:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.539391:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.539397:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=glibc-hwcaps, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.539429:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: glibc-hwcaps,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.539484:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.539491:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.539500:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.539520:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007232:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.539611:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.545956:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007232:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.545980:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.545992:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.545999:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546006:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546014:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546021:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.546026:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546034:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546041:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546078:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.546085:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546092:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e0ea lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.546116:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913007232/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.546132:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.546219:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.546233:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.546238:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.546258:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.546310:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546318:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.546328:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.546351:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007360:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.546501:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.550248:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007360:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.550272:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550286:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550292:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550299:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550306:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550311:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1c00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.550316:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550322:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550332:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550339:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.550346:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550352:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e0f1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.550379:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913007360/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.550394:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.550492:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.550507:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.550511:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.550531:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.550583:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550590:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.550598:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.550620:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007488:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.550990:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.554017:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007488:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.554042:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554120:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554127:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554135:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554142:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554148:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.554154:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554161:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554168:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554175:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.554182:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554188:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e0f8 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.554211:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913007488/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.554226:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.554312:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.554325:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.554329:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.554352:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.554403:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554422:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.554432:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.554456:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007616:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.554553:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.558542:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007616:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.558569:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558584:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558591:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558598:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558606:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558613:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.558619:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558627:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558633:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558640:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.558648:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558655:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e0ff lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.558691:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913007616/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.558721:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.558811:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.558829:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.558834:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=tls, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.558856:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: tls,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.558906:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558914:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.558922:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.558946:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007744:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.559073:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.562303:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007744:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.562330:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562344:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562350:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562358:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562365:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562371:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1c00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.562376:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562383:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562390:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562398:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.562405:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562425:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e106 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.562458:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913007744/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.562473:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.562570:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.562587:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.562591:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.562611:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.562675:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562683:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.562693:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.562726:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007872:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.562823:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.565825:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913007872:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.565848:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565860:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565866:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565872:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565880:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565886:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1a00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.565890:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565896:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565902:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565908:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.565915:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.565920:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1a00/0x410246667097e10d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.565944:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913007872/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.565958:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.566085:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.566102:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.566107:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.566126:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.566171:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.566178:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.566188:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.566209:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913008000:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.566293:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.569115:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913008000:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.569139:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569152:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569160:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569167:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569175:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569181:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.569185:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569192:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569198:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569205:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.569212:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569219:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e114 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.569241:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913008000/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.569256:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.569333:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.569346:0:21970:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), flags=17 00000080:00200000:1.0:1769074938.569351:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=x86_64, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=0 00000002:00010000:1.0:1769074938.569368:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: x86_64,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074938.569423:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569432:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.569442:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.569463:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913008128:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.569648:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.576886:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913008128:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.576913:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576927:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576939:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576946:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576955:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576961:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1000 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.576966:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576973:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576980:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.576987:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.576994:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.577000:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e11b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.577028:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=7, status=0 req@ffff9350062a6680 x1855008913008128/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.577042:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.577156:0:21970:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.577172:0:21970:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=libpthread.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), file ffff9350124e7380, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1769074938.577180:0:21970:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=libpthread.so.0, dir=[0x200000401:0x21:0x0](ffff9350115ab248), intent=open 00000002:00010000:1.0:1769074938.577200:0:21970:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libpthread.so.0,[0x0:0x0:0x0]) in obj [0x200000401:0x21:0x0], intent: open flags 0100001 00010000:00010000:1.0:1769074938.577256:0:21970:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.577263:0:21970:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000020:00100000:1.0:1769074938.577275:0:21970:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074938.577282:0:21970:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6680 x1855008913008384/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074938.577291:0:21970:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00000100:00100000:1.0:1769074938.577313:0:21970:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913008384:192.168.204.121@tcp:101:ls.0 00000100:00100000:1.0:1769074938.577582:0:21970:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350123cdc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074938.580904:0:21970:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ls:7afd476b-94ee-4c94-b080-d15824f3ae56:21970:1855008913008384:192.168.204.121@tcp:101:ls.0 00010000:00010000:1.0:1769074938.580927:0:21970:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580941:0:21970:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580948:0:21970:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580955:0:21970:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580963:0:21970:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580971:0:21970:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074938.580976:0:21970:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580983:0:21970:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580990:0:21970:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.580996:0:21970:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074938.581002:0:21970:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00010000:00010000:1.0:1769074938.581008:0:21970:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e122 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x21:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 21970 initiator: MDT0 00000002:00100000:1.0:1769074938.581033:0:21970:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff9350062a6680 x1855008913008384/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 656/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ls.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074938.581069:0:21970:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x21:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074938.627076:0:21970:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000401:0x21:0x0](ffff9350115ab248) pos/size18446744073709551614/512 32bit_api 0 00000080:00200008:1.0:1769074938.627168:0:21970:0:(file.c:424:ll_file_release()) START file d13:[0x200000401:0x21:0x0](ffff9350115ab248), flags 304000 00010000:00010000:1.0:1769074938.627198:0:21970:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e129 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x21:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97a1c expref: -99 pid: 21970 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074939.060020:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074940.081918:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074944.178318:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074945.206954:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074946.118687:0:22535:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074946.118719:0:22535:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074946.118742:0:22535:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.118760:0:22535:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074946.118769:0:22535:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074946.118777:0:22535:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074946.118781:0:22535:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097d870 00000080:00010000:1.0:1769074946.118789:0:22535:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d870 from it ffffa8e541f03be0 00010000:00010000:1.0:1769074946.118793:0:22535:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.118801:0:22535:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074946.118819:0:22535:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074946.118838:0:22535:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d14aa.sanityn, dir=[0x200000007:0x1:0x0](ffff935011534b08), flags=1536 00000080:00200000:1.0:1769074946.118841:0:22535:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074946.118846:0:22535:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074946.118852:0:22535:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.118859:0:22535:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074946.118864:0:22535:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074946.118867:0:22535:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074946.118869:0:22535:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097d870 00000080:00010000:1.0:1769074946.118873:0:22535:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d870 from it ffffa8e541f03ce0 00010000:00010000:1.0:1769074946.118876:0:22535:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.118883:0:22535:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074946.118890:0:22535:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 3 00000080:00200000:1.0:1769074946.118895:0:22535:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074946.118900:0:22535:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074946.118905:0:22535:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.118912:0:22535:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074946.118917:0:22535:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074946.118921:0:22535:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074946.118923:0:22535:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097d870 00000080:00010000:1.0:1769074946.118926:0:22535:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097d870 from it ffffa8e541f03dc8 00010000:00010000:1.0:1769074946.118929:0:22535:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.118934:0:22535:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074946.118940:0:22535:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 3 00000080:00200000:1.0:1769074946.118945:0:22535:0:(namei.c:2175:do_mkdir()) VFS Op:name=d14aa.sanityn, dir=[0x200000007:0x1:0x0](ffff935011534b08) 00010000:00010000:1.0:1769074946.118979:0:22535:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.119041:0:22535:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.119080:0:22535:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074946.119085:0:22535:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097d870 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf96076 expref: -99 pid: 21811 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074946.119110:0:22535:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074946.119128:0:22535:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6680 x1855008913010304/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 560/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074946.119156:0:22535:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:22535:1855008913010304:192.168.204.121@tcp:36:mkdir.0 00000100:00100000:1.0:1769074946.119273:0:22535:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693ec00 going to sleep for 11 seconds 00010000:00010000:1.0:1769074946.125819:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074946.125829:0:5554:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.125841:0:5554:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216f400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074946.125846:0:5554:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.125854:0:5554:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.125865:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.125940:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.125947:0:5554:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074946.125975:0:5554:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.125982:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f400/0x410246667097daae lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf9668f expref: -99 pid: 21964 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074946.125996:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.125998:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.126001:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074946.126136:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913010432:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074946.131793:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913010432:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074946.132938:0:22535:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:22535:1855008913010304:192.168.204.121@tcp:36:mkdir.0 00000080:00200000:1.0:1769074946.133006:0:22535:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000401:0x24:0x0](ffff9350115af448) 00000080:00200000:1.0:1769074946.133013:0:22535:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x24:0x0], updating i_size 512 00000080:00200000:1.0:1769074946.162361:0:22538:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074946.162396:0:22538:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074946.162415:0:22538:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589934599/1 (0 0) 00010000:00010000:1.0:1769074946.162487:0:22538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e137 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22538 initiator: MDT0 00010000:00010000:1.0:1769074946.162498:0:22538:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e137 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22538 initiator: MDT0 00010000:00010000:1.0:1769074946.162509:0:22538:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e137 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22538 initiator: MDT0 00000100:00100000:1.0:1769074946.162540:0:22538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913010560:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074946.162676:0:22538:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693ec00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074946.215886:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913011328:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074946.225228:0:2363:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 6/2,0 mode: --/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.225254:0:2363:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 6/2,0 mode: --/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.225272:0:2363:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 6/2,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.225520:0:2363:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 6/2,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074946.225541:0:2363:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 5/2,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074946.225553:0:2363:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1769074946.225585:0:2363:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 6/2,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.225594:0:2363:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 5/1,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.225602:0:2363:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 5/1,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.225616:0:2363:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 4/0,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074946.225631:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913011328:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074946.244435:0:14430:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074946.244443:0:14430:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 2/0,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.244453:0:14430:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216f400 already unused, calling callback (ffffffffc137c0e0) 00010000:00010000:1.0:1769074946.244459:0:14430:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 3/0,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074946.244482:0:14430:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074946.244497:0:14430:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074946.244549:0:14430:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 2/0,0 mode: --/PR res: [0x240000400:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.244556:0:14430:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074946.244590:0:14430:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 1/0,0 mode: --/PR res: [0x240000400:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.244598:0:14430:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93504216f400/0x410246667097e15a lrc: 0/0,0 mode: --/PR res: [0x240000400:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97acb expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.244613:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.244615:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.244618:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074946.244639:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913011584:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074946.254298:0:22538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913011456:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074946.254332:0:22538:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,1 mode: --/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->143359] (req 0->143359) gid 0 flags: 0x10000000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.254348:0:22538:0:(ldlm_request.c:274:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,1 mode: --/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->143359] (req 0->143359) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.254357:0:22538:0:(ldlm_request.c:303:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,1 mode: --/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->143359] (req 0->143359) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.254486:0:22538:0:(ldlm_request.c:186:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.254494:0:22538:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.254509:0:22538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x10000000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074946.254522:0:22538:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x10020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.254533:0:22538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.254546:0:22538:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074946.254588:0:22538:0:(vvp_io.c:1312:vvp_io_write_start()) f14aa.sanityn: write [0, 143296) append 0 append lockpos 0 00000080:00200000:1.0:1769074946.254609:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.254670:0:22538:0:(rw26.c:912:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1769074946.254686:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000523fd37f[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.254689:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000001ca84ef4 17ffffc0001001 3:0 ffff9350109da360 0 lru 00000080:00200000:1.0:1769074946.254697:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000308a0301 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.254699:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000523fd37f 00000080:00200000:1.0:1769074946.254700:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1769074946.254703:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.254731:0:22538:0:(rw26.c:912:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:1.0:1769074946.254735:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000041aa1b32[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.254738:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000821da543 17ffffc0001001 3:0 ffff9350109da798 1 lru 00000080:00200000:1.0:1769074946.254744:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000005a4721aa 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.254746:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000041aa1b32 00000080:00200000:1.0:1769074946.254747:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 2. 00000080:00200000:1.0:1769074946.254749:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.254764:0:22538:0:(rw26.c:912:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:1.0:1769074946.254768:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000e1f35e6d[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.254770:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @0000000088e75678 17ffffc0001001 3:0 ffff9350109da510 2 lru 00000080:00200000:1.0:1769074946.254775:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000128b688c 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.254777:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000e1f35e6d 00000080:00200000:1.0:1769074946.254778:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 3. 00000080:00200000:1.0:1769074946.254779:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.254793:0:22538:0:(rw26.c:912:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:1.0:1769074946.254797:0:22538:0:(rw26.c:969:ll_write_end()) page@000000003e4b5342[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.254800:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000007b6f0fbe 17ffffc0001001 3:0 ffff9350109dabd0 3 lru 00000080:00200000:1.0:1769074946.254805:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000007fb13164 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.254807:0:22538:0:(rw26.c:969:ll_write_end()) end page@000000003e4b5342 00000080:00200000:1.0:1769074946.254809:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 4. 00000080:00200000:1.0:1769074946.254811:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255388:0:22538:0:(rw26.c:912:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255406:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000076f41632[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255409:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000009fe1019c 17ffffc0001001 3:0 ffff9350109dad80 4 lru 00000080:00200000:1.0:1769074946.255415:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000ba8054b5 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255417:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000076f41632 00000080:00200000:1.0:1769074946.255419:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 5. 00000080:00200000:1.0:1769074946.255421:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255441:0:22538:0:(rw26.c:912:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255445:0:22538:0:(rw26.c:969:ll_write_end()) page@000000003ba63c52[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255448:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000df00713d 17ffffc0001001 3:0 ffff9350109da000 5 lru 00000080:00200000:1.0:1769074946.255453:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000006e71e22b 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255455:0:22538:0:(rw26.c:969:ll_write_end()) end page@000000003ba63c52 00000080:00200000:1.0:1769074946.255456:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 6. 00000080:00200000:1.0:1769074946.255458:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255471:0:22538:0:(rw26.c:912:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255475:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000014ee585[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255478:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000a9b6dcbd 17ffffc0001001 3:0 ffff9350109da5e8 6 lru 00000080:00200000:1.0:1769074946.255483:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000a9f77d3f 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255485:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000014ee585 00000080:00200000:1.0:1769074946.255486:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 7. 00000080:00200000:1.0:1769074946.255487:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255499:0:22538:0:(rw26.c:912:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255502:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000f61897a0[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255505:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000009fdea7d0 17ffffc0001001 3:0 ffff9350109da288 7 lru 00000080:00200000:1.0:1769074946.255510:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@0000000096716726 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255512:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000f61897a0 00000080:00200000:1.0:1769074946.255513:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 8. 00000080:00200000:1.0:1769074946.255515:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255525:0:22538:0:(rw26.c:912:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255529:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000cbe8a3c9[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255532:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000fd78a34d 17ffffc0001001 3:0 ffff9350109da0d8 8 lru 00000080:00200000:1.0:1769074946.255537:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000003c3809e8 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255538:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000cbe8a3c9 00000080:00200000:1.0:1769074946.255539:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 9. 00000080:00200000:1.0:1769074946.255541:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255552:0:22538:0:(rw26.c:912:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255556:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000046ca206e[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255558:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000b8aea710 17ffffc0001001 3:0 ffff9350109dae58 9 lru 00000080:00200000:1.0:1769074946.255563:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000bafc97c4 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255564:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000046ca206e 00000080:00200000:1.0:1769074946.255565:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 10. 00000080:00200000:1.0:1769074946.255567:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255591:0:22538:0:(rw26.c:912:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255595:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000089579d94[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255598:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000003c94ea7d 17ffffc0001001 3:0 ffff9350109da948 10 lru 00000080:00200000:1.0:1769074946.255603:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@0000000014be4103 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255605:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000089579d94 00000080:00200000:1.0:1769074946.255606:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 11. 00000080:00200000:1.0:1769074946.255608:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255622:0:22538:0:(rw26.c:912:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255626:0:22538:0:(rw26.c:969:ll_write_end()) page@000000004b32cec2[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255628:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000cccfe1e6 17ffffc0001001 3:0 ffff9350109da438 11 lru 00000080:00200000:1.0:1769074946.255633:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000ac055f21 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255635:0:22538:0:(rw26.c:969:ll_write_end()) end page@000000004b32cec2 00000080:00200000:1.0:1769074946.255636:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 12. 00000080:00200000:1.0:1769074946.255638:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255649:0:22538:0:(rw26.c:912:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255653:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000046a376a5[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255656:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000006a9800bf 17ffffc0001001 3:0 ffff9350109da6c0 12 lru 00000080:00200000:1.0:1769074946.255661:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000007ad6415c 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255663:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000046a376a5 00000080:00200000:1.0:1769074946.255665:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 13. 00000080:00200000:1.0:1769074946.255666:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255678:0:22538:0:(rw26.c:912:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255681:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000022941b1e[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255684:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000b4ccc98e 17ffffc0001001 3:0 ffff9350109daaf8 13 lru 00000080:00200000:1.0:1769074946.255689:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000aca82b4c 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255692:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000022941b1e 00000080:00200000:1.0:1769074946.255693:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 14. 00000080:00200000:1.0:1769074946.255694:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255714:0:22538:0:(rw26.c:912:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255718:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000b0cd5e75[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255720:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000007a73a566 17ffffc0001021 2:0 ffff9350109daca8 14 lru 00000080:00200000:1.0:1769074946.255725:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000102b3cd0 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255727:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000b0cd5e75 00000080:00200000:1.0:1769074946.255728:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 15. 00000080:00200000:1.0:1769074946.255730:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255741:0:22538:0:(rw26.c:912:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255745:0:22538:0:(rw26.c:969:ll_write_end()) page@000000004bc98d35[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255748:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000005d4e67e9 17ffffc0001001 3:0 ffff9350109daa20 15 lru 00000080:00200000:1.0:1769074946.255753:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000de083724 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255755:0:22538:0:(rw26.c:969:ll_write_end()) end page@000000004bc98d35 00000080:00200000:1.0:1769074946.255756:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 16. 00000080:00200000:1.0:1769074946.255757:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255769:0:22538:0:(rw26.c:912:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255772:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000079969d24[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255775:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000396b0296 17ffffc0001001 3:0 ffff9350109da1b0 16 lru 00000080:00200000:1.0:1769074946.255780:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000006755919c 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255782:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000079969d24 00000080:00200000:1.0:1769074946.255783:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 17. 00000080:00200000:1.0:1769074946.255785:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.255799:0:22538:0:(rw26.c:912:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:1.0:1769074946.255803:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000551e6400[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.255805:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000005cfd522e 17ffffc0001001 3:0 ffff9350109da870 17 lru 00000080:00200000:1.0:1769074946.255811:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000f5106a10 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.255813:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000551e6400 00000080:00200000:1.0:1769074946.255814:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 18. 00000080:00200000:1.0:1769074946.255816:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256177:0:22538:0:(rw26.c:912:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256182:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000b508a0b7[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256185:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000006e68d081 17ffffc0001001 3:0 ffff935007e08e58 18 lru 00000080:00200000:1.0:1769074946.256190:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000c42cb2c7 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256192:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000b508a0b7 00000080:00200000:1.0:1769074946.256194:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 19. 00000080:00200000:1.0:1769074946.256196:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256214:0:22538:0:(rw26.c:912:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256218:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000046dccfe7[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256220:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000009242bf2b 17ffffc0001001 3:0 ffff935007e080d8 19 lru 00000080:00200000:1.0:1769074946.256225:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000c7905d78 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256227:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000046dccfe7 00000080:00200000:1.0:1769074946.256228:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 20. 00000080:00200000:1.0:1769074946.256230:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256244:0:22538:0:(rw26.c:912:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256248:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000008227474[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256250:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000005a5dfe41 17ffffc0001001 3:0 ffff935007e08288 20 lru 00000080:00200000:1.0:1769074946.256255:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000000023c0d4 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256257:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000008227474 00000080:00200000:1.0:1769074946.256258:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 21. 00000080:00200000:1.0:1769074946.256260:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256274:0:22538:0:(rw26.c:912:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256278:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000a32dc9a3[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256281:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000790520bd 17ffffc0001001 3:0 ffff935007e085e8 21 lru 00000080:00200000:1.0:1769074946.256286:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000c5835da9 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256289:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000a32dc9a3 00000080:00200000:1.0:1769074946.256290:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 22. 00000080:00200000:1.0:1769074946.256292:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256307:0:22538:0:(rw26.c:912:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256311:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000cf1e9267[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256314:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000f6fbbbbc 17ffffc0001001 3:0 ffff935007e08000 22 lru 00000080:00200000:1.0:1769074946.256318:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000918cd7cd 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256320:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000cf1e9267 00000080:00200000:1.0:1769074946.256321:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 23. 00000080:00200000:1.0:1769074946.256323:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256336:0:22538:0:(rw26.c:912:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256340:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000947a0738[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256342:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000abc64177 17ffffc0001001 3:0 ffff935007e08d80 23 lru 00000080:00200000:1.0:1769074946.256348:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000005a76b5ab 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256351:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000947a0738 00000080:00200000:1.0:1769074946.256352:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 24. 00000080:00200000:1.0:1769074946.256354:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256370:0:22538:0:(rw26.c:912:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256374:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000acd42cfc[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256377:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000281abde6 17ffffc0001001 3:0 ffff935007e08bd0 24 lru 00000080:00200000:1.0:1769074946.256382:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000b8641563 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256384:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000acd42cfc 00000080:00200000:1.0:1769074946.256384:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 25. 00000080:00200000:1.0:1769074946.256386:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256462:0:22538:0:(rw26.c:912:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256467:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000a04dc877[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256470:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000c8ab6168 17ffffc0001001 3:0 ffff935007e08510 25 lru 00000080:00200000:1.0:1769074946.256476:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000711e1c08 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256478:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000a04dc877 00000080:00200000:1.0:1769074946.256479:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 26. 00000080:00200000:1.0:1769074946.256480:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256495:0:22538:0:(rw26.c:912:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256498:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000ec99b1f4[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256501:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000199e90ba 17ffffc0001001 3:0 ffff935007e08360 26 lru 00000080:00200000:1.0:1769074946.256506:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000cb53a965 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256508:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000ec99b1f4 00000080:00200000:1.0:1769074946.256509:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 27. 00000080:00200000:1.0:1769074946.256511:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256523:0:22538:0:(rw26.c:912:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256527:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000410f028f[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256530:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000006f0d5367 17ffffc0001001 3:0 ffff935007e08798 27 lru 00000080:00200000:1.0:1769074946.256535:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000003da6bdc5 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256537:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000410f028f 00000080:00200000:1.0:1769074946.256538:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 28. 00000080:00200000:1.0:1769074946.256539:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256550:0:22538:0:(rw26.c:912:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256553:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000040d6d867[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256556:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000005d22ff61 17ffffc0001001 3:0 ffff935007e08870 28 lru 00000080:00200000:1.0:1769074946.256561:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000b4b72bc5 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256563:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000040d6d867 00000080:00200000:1.0:1769074946.256565:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 29. 00000080:00200000:1.0:1769074946.256566:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256592:0:22538:0:(rw26.c:912:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256596:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000085e0ac0e[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256599:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000da9c9f55 17ffffc0001021 2:0 ffff935007e081b0 29 lru 00000080:00200000:1.0:1769074946.256604:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000008bd5e72a 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256607:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000085e0ac0e 00000080:00200000:1.0:1769074946.256608:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 30. 00000080:00200000:1.0:1769074946.256609:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256622:0:22538:0:(rw26.c:912:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256625:0:22538:0:(rw26.c:969:ll_write_end()) page@000000000e0bb216[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256628:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000d7cca989 17ffffc0001001 3:0 ffff935007e08a20 30 lru 00000080:00200000:1.0:1769074946.256633:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000afd7cfa4 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256636:0:22538:0:(rw26.c:969:ll_write_end()) end page@000000000e0bb216 00000080:00200000:1.0:1769074946.256637:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 31. 00000080:00200000:1.0:1769074946.256639:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256652:0:22538:0:(rw26.c:912:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256656:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000029c5d06a[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256659:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @000000007f7e9de4 17ffffc0001001 3:0 ffff935007e08ca8 31 lru 00000080:00200000:1.0:1769074946.256664:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000007da82ca9 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256666:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000029c5d06a 00000080:00200000:1.0:1769074946.256667:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 32. 00000080:00200000:1.0:1769074946.256669:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256682:0:22538:0:(rw26.c:912:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256685:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000067071cbd[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256688:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000ab408c6d 17ffffc0001001 3:0 ffff935007e08af8 32 lru 00000080:00200000:1.0:1769074946.256693:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@0000000023a08f6c 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256695:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000067071cbd 00000080:00200000:1.0:1769074946.256696:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 33. 00000080:00200000:1.0:1769074946.256698:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:1.0:1769074946.256711:0:22538:0:(rw26.c:912:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:1.0:1769074946.256714:0:22538:0:(rw26.c:969:ll_write_end()) page@00000000888f21f8[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256717:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @00000000badde10a 17ffffc0001001 3:0 ffff935007e086c0 33 lru 00000080:00200000:1.0:1769074946.256721:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@00000000b43ce00d 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256723:0:22538:0:(rw26.c:969:ll_write_end()) end page@00000000888f21f8 00000080:00200000:1.0:1769074946.256724:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 34. 00000080:00200000:1.0:1769074946.256726:0:22538:0:(rw26.c:729:ll_write_begin()) Writing 34 of 0 to 4032 bytes 00000080:00200000:1.0:1769074946.256749:0:22538:0:(rw26.c:912:ll_write_end()) pos 139264, len 4032, copied 4032 00000080:00200000:1.0:1769074946.256753:0:22538:0:(rw26.c:969:ll_write_end()) page@0000000018793bc2[3 00000000f1d839da 2 1 000000006bf70109] 00000080:00200000:1.0:1769074946.256756:0:22538:0:(rw26.c:969:ll_write_end()) vmpage @0000000002e58a27 17ffffc0001001 3:0 ffff935007e08438 34 lru 00000080:00200000:1.0:1769074946.256761:0:22538:0:(rw26.c:969:ll_write_end()) osc-page@000000000d595765 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 0000000001c2e547 00000000a8aa2a64 > 3< 0 0 > 4< 0 0 8 17432576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074946.256763:0:22538:0:(rw26.c:969:ll_write_end()) end page@0000000018793bc2 00000080:00200000:1.0:1769074946.256764:0:22538:0:(rw26.c:969:ll_write_end()) queued page: 35. 00000080:00200000:1.0:1769074946.256768:0:22538:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 35, from 0, to 4032 prio 0 00000008:00010000:1.0:1769074946.256798:0:22538:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000931f19d9 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074946.256916:0:22538:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff93501158abb8, count 15, dirtied 15 00000080:00200000:1.0:1769074946.257286:0:22538:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff93501158abb8, count 15, dirtied 15 00000080:00200000:1.0:1769074946.257299:0:22538:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff93501158abb8, count 5, dirtied 5 00000080:00200000:1.0:1769074946.257304:0:22538:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 35 pages 143296 bytes, tot: 143296 00000080:00200000:1.0:1769074946.257308:0:22538:0:(vvp_io.c:1390:vvp_io_write_start()) f14aa.sanityn: write bytes 0, result: 143296 00000020:00010000:1.0:1769074946.257329:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074946.257333:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074946.257337:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93501258f230 (ffff935005339bd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074946.257345:0:22538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.257359:0:22538:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074946.257378:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074946.257381:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93501258f230 (ffff9350053393f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074946.257473:0:22538:0:(vvp_io.c:518:vvp_io_advance()) advancing 143296 bytes 00000080:00200000:1.0:1769074946.257482:0:22538:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 34] 00000080:00200000:1.0:1769074946.257493:0:22538:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x25:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074946.257497:0:22538:0:(file.c:2142:ll_file_io_generic()) f14aa.sanityn: 2 io complete with rc: 0, result: 143296, restart: 0 00000080:00200000:1.0:1769074946.257507:0:22538:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x25:0x0] updating i_size 143296 i_blocks 1 00000080:00200000:1.0:1769074946.257513:0:22538:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 143296 00000080:00200008:1.0:1769074946.257555:0:22538:0:(llite_lib.c:2314:ll_setattr_raw()) START file f14aa.sanityn:[0x200000401:0x25:0x0](ffff93501158aa08) current size 143296, valid attrs 101f0, mode ffff, uid 130503552, gid 131072, new size 18446624570670008192, atime 1764139040.000000000, mtime 1674053537.000000000, ctime 1769074946.000000000 00010000:00010000:1.0:1769074946.257597:0:22538:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097e153 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97abd expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074946.257624:0:22538:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x25:0x0]: losing layout lock 00010000:00010000:1.0:1769074946.257679:0:22538:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097e153 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97abd expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.257686:0:22538:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074946.257689:0:22538:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e800/0x410246667097e153 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97abd expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074946.257718:0:22538:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074946.257732:0:22538:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7800 x1855008913011840/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074946.257758:0:22538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913011840:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074946.258185:0:22538:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693e900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074946.270343:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913011584:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074946.271173:0:22538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913011840:192.168.204.121@tcp:36:cp.0 00000080:00200000:1.0:1769074946.271216:0:22538:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x25:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:1.0:1769074946.271234:0:22538:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0x25:0x0](ffff93501158aa08) 00000002:00010000:1.0:1769074946.271245:0:22538:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x25:0x0]) in obj [0x200000401:0x25:0x0], intent: layout flags 0 00010000:00010000:1.0:1769074946.271293:0:22538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x25:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22538 initiator: MDT0 00010000:00010000:1.0:1769074946.271301:0:22538:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22538 initiator: MDT0 00010000:00010000:1.0:1769074946.271308:0:22538:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22538 initiator: MDT0 00000100:00100000:1.0:1769074946.271326:0:22538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913011968:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074946.271472:0:22538:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693e900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074946.277152:0:22538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913011968:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074946.277196:0:22538:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.277213:0:22538:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1769074946.277229:0:22538:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9350062a6a00 x1855008913011968/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 376/400 e 0 to 0 dl 1769074962 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074946.277239:0:22538:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00000002:00010000:1.0:1769074946.277249:0:22538:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.277261:0:22538:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 16 res 8589935617/37 (0 0) 00000080:00200000:1.0:1769074946.277267:0:22538:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0x25:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1769074946.277611:0:22538:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x25:0x0](ffff93501158aa08) for lock 0x410246667097e168 00000080:00010000:1.0:1769074946.277623:0:22538:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e168 from it ffffa8e541523a78 00010000:00010000:1.0:1769074946.277628:0:22538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.277638:0:22538:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.277669:0:22538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.277683:0:22538:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1769074946.277692:0:22538:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000401:0x25:0x0](00000000c0ea77e3) being reconfigured ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074946.277735:0:22538:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x25:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:1.0:1769074946.277739:0:22538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.277743:0:22538:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00020000:00200000:1.0:1769074946.277763:0:22538:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074946.277775:0:22538:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1769074946.277788:0:22538:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074946.277792:0:22538:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c430, osc lock ffff935010592f70, flags 0x0 00010000:00010000:1.0:1769074946.277808:0:22538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.277814:0:22538:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.277826:0:22538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.277839:0:22538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.277845:0:22538:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074946.277940:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074946.277961:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913012096:192.168.204.121@tcp:2:cp.0 00000020:00010000:1.0:1769074946.285327:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074946.285331:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074946.285337:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5c430 (ffff935005339bd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074946.285349:0:22538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.285362:0:22538:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074946.285385:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074946.285388:0:22538:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5c430 (ffff9350053393f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074946.285413:0:22538:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x25:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:1.0:1769074946.285429:0:22538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff935002a40c00 uarg=0000000000000000 00000002:00000080:1.0:1769074946.285440:0:22538:0:(mdc_request.c:2279:mdc_iocontrol()) lustre-MDT0000-mdc-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff935002a40c00 uarg=0000000000000000 00000020:00100000:1.0:1769074946.285472:0:22538:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:1.0:1769074946.285481:0:22538:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5f80 x1855008913012352/t0(0) o55->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074946.285509:0:22538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913012352:192.168.204.121@tcp:55:cp.0 00000100:00100000:1.0:1769074946.285604:0:22538:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693e900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074946.290347:0:22538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913012352:192.168.204.121@tcp:55:cp.0 00000080:00200008:1.0:1769074946.290407:0:22538:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f14aa.sanityn:[0x200000401:0x25:0x0](ffff93501158aa08) current size 143296, valid attrs 101f0, mode ffff, uid 130503552, gid 131072, new size 18446624570670008192, atime 1764139040.000000000, mtime 1674053537.000000000, ctime 1769074946.000000000, rc 0 00000020:00100000:1.0:1769074946.290550:0:22538:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074946.290559:0:22538:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5f80 x1855008913012608/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 544/440 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074946.290601:0:22538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913012608:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074946.290690:0:22538:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693e900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074946.297326:0:22538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913012608:192.168.204.121@tcp:36:cp.0 00000080:00200008:1.0:1769074946.297415:0:22538:0:(file.c:424:ll_file_release()) START file f14aa.sanityn:[0x200000401:0x25:0x0](ffff93501158aa08), flags 100001 00000002:00100000:1.0:1769074946.297479:0:22538:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350103f9c00 x1855008913011200/t4294973363(4294973363) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/664 e 0 to 0 dl 1769074962 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074946.297491:0:22538:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074946.297495:0:22538:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008913012864/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074946.297518:0:22538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913012864:192.168.204.121@tcp:35:cp.0 00000100:00100000:1.0:1769074946.297607:0:22538:0:(client.c:2690:ptlrpc_set_wait()) set ffff93500693e900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074946.304999:0:22538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:22538:1855008913012864:192.168.204.121@tcp:35:cp.0 00000002:00100000:1.0:1769074946.305042:0:22538:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9350103f9c00 x1855008913011200/t4294973363(4294973363) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/664 e 0 to 0 dl 1769074962 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1769074946.480880:0:14429:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074946.480901:0:14429:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800420000020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.480924:0:14429:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935012356a00 already unused, calling callback (ffffffffc137c0e0) 00010000:00010000:1.0:1769074946.480934:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800428400020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074946.480963:0:14429:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 00000000931f19d9 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800429400020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074946.480980:0:14429:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 00000000931f19d9 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800429400020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.481111:0:14429:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 5/0,0 mode: PW/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x800429400020000 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074946.481887:0:14429:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9350062a7480 - object 0x240000400:10 offset 0<>143296 00010000:00010000:1.0:1769074946.486884:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074946.486897:0:5554:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097e1ae lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x48bf02eaaaf97b9d expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.486910:0:5554:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93504216f800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074946.486915:0:5554:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097e1ae lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x48bf02eaaaf97b9d expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.486931:0:5554:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097e1ae lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x48bf02eaaaf97b9d expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.486943:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097e1ae lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97b9d expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.486980:0:5554:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012356e00/0x410246667097e18b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97b73 expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.487042:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097e1ae lrc: 2/0,0 mode: --/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf97b9d expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.487076:0:5554:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074946.491676:0:5554:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097e1ae lrc: 1/0,0 mode: --/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf97b9d expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.491701:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93504216f800/0x410246667097e1ae lrc: 0/0,0 mode: --/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf97b9d expref: -99 pid: 22548 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074946.491732:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.491735:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.491739:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074946.510352:0:14429:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074946.510478:0:14429:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074946.510549:0:14429:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 2/0,0 mode: --/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x804c69400020020 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.510558:0:14429:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074946.510597:0:14429:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 1/0,0 mode: --/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x804c69400020020 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.510605:0:14429:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e161 lrc: 0/0,0 mode: --/PW res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->143359) gid 0 flags: 0x804c69400020020 nid: local remote: 0x48bf02eaaaf97ad2 expref: -99 pid: 22538 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074946.510630:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.510632:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074946.510634:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074946.510642:0:2363:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074946.510657:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913015040:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074946.524730:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913015040:192.168.204.121@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074948.275381:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074949.305625:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074952.373678:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074952.931757:0:23104:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanityn test 14ab: open(RDWR) of executing file returns -ETXTBSY ========================================================== 04:42:31 (1769074951) 00010000:00010000:1.0:1769074953.261675:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074953.261685:0:5554:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012357800/0x410246667097e176 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x48bf02eaaaf97b1f expref: -99 pid: 22548 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.261695:0:5554:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935012357800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074953.261702:0:5554:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012357800/0x410246667097e176 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97b1f expref: -99 pid: 22548 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.261712:0:5554:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012357800/0x410246667097e176 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97b1f expref: -99 pid: 22548 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.261721:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012357800/0x410246667097e176 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x48bf02eaaaf97b1f expref: -99 pid: 22548 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.261793:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012357800/0x410246667097e176 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97b1f expref: -99 pid: 22548 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.261806:0:5554:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074953.261831:0:5554:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012357800/0x410246667097e176 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97b1f expref: -99 pid: 22548 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.261839:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935012357800/0x410246667097e176 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97b1f expref: -99 pid: 22548 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.261853:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074953.261855:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074953.261857:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074953.261866:0:2363:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074953.261883:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913017856:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074953.269694:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913017856:192.168.204.121@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074953.397491:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074953.411654:0:23124:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018112:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074953.411805:0:23124:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001254840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074953.418652:0:23124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018112:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074953.418688:0:23124:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418714:0:23124:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418723:0:23124:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418747:0:23124:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418760:0:23124:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418774:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418782:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418790:0:23124:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074953.418802:0:23124:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff9350103f8380 x1855008913018112/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074969 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074953.418815:0:23124:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418828:0:23124:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/39 (0 0) 00000080:00200000:1.0:1769074953.418835:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074953.418863:0:23124:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x27:0x0], updating i_size 512 00000080:00200000:1.0:1769074953.418866:0:23124:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x27:0x0](ffff9350115f63c8): rc = 0 00000080:00010000:1.0:1769074953.418873:0:23124:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27:0x0](ffff9350115f63c8) for lock 0x410246667097e1c3 00000080:00010000:1.0:1769074953.418885:0:23124:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x27:0x0](ffff9350115f63c8) 00000080:00010000:1.0:1769074953.418890:0:23124:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1c3 from it ffffa8e541f33a28 00010000:00010000:1.0:1769074953.418893:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.418902:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074953.418943:0:23124:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27:0x0](ffff9350115f63c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074953.418964:0:23124:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=sleep, dir=[0x200000401:0x27:0x0](ffff9350115f63c8), flags=9 00000080:00200000:1.0:1769074953.418968:0:23124:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x27:0x0](ffff9350115f63c8), intent=getattr 00000002:00010000:1.0:1769074953.418988:0:23124:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x0:0x0:0x0]) in obj [0x200000401:0x27:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074953.419038:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.419045:0:23124:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.419248:0:23124:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00000100:00100000:1.0:1769074953.419271:0:23124:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018240:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074953.419355:0:23124:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001254840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074953.431754:0:23124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018240:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074953.431783:0:23124:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.431800:0:23124:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.431807:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.431820:0:23124:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 3/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.431828:0:23124:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 4/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.431834:0:23124:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1800 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074953.431839:0:23124:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 4/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.431846:0:23124:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 4/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.432326:0:23124:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 5/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.432337:0:23124:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074953.432346:0:23124:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 3/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.432353:0:23124:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ca lrc: 0/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23124 initiator: MDT0 00000002:00100000:1.0:1769074953.432375:0:23124:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9350062a5880 x1855008913018240/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074969 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074953.432392:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x27:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074953.432423:0:23124:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/39 (0 0) 00000080:00200000:1.0:1769074953.432534:0:23124:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074953.432554:0:23124:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074953.432564:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.432584:0:23124:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074953.432590:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074953.432596:0:23124:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074953.432599:0:23124:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e1bc 00000080:00010000:1.0:1769074953.432604:0:23124:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1bc from it ffffa8e541f33b58 00010000:00010000:1.0:1769074953.432608:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.432616:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074953.432632:0:23124:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074953.432639:0:23124:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14ab.sanityn, flags=25 00000080:00200000:1.0:1769074953.432645:0:23124:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27:0x0](ffff9350115f63c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074953.432660:0:23124:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=sleep, dir=[0x200000401:0x27:0x0](ffff9350115f63c8), flags=9 00000080:00200000:1.0:1769074953.432664:0:23124:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x27:0x0](ffff9350115f63c8), intent=getattr 00000002:00010000:1.0:1769074953.432676:0:23124:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x0:0x0:0x0]) in obj [0x200000401:0x27:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074953.432729:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.432737:0:23124:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.432748:0:23124:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00000100:00100000:1.0:1769074953.432771:0:23124:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018368:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074953.432928:0:23124:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001254840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074953.444875:0:23124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018368:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074953.444902:0:23124:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444917:0:23124:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444925:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444933:0:23124:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444941:0:23124:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444948:0:23124:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa1e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074953.444953:0:23124:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444959:0:23124:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444966:0:23124:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444974:0:23124:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074953.444980:0:23124:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.444986:0:23124:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1d1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23124 initiator: MDT0 00000002:00100000:1.0:1769074953.445010:0:23124:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9350062a5880 x1855008913018368/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074969 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074953.445027:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x27:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074953.445651:0:23124:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/39 (0 0) 00000080:00200000:1.0:1769074953.445786:0:23124:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074953.445804:0:23124:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074953.445815:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.445829:0:23124:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074953.445836:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074953.445842:0:23124:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074953.445845:0:23124:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e1bc 00000080:00010000:1.0:1769074953.445850:0:23124:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1bc from it ffffa8e541f33bf8 00010000:00010000:1.0:1769074953.445854:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.445863:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074953.445879:0:23124:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074953.445887:0:23124:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14ab.sanityn, flags=25 00000080:00200000:1.0:1769074953.445893:0:23124:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27:0x0](ffff9350115f63c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074953.445907:0:23124:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27:0x0](ffff9350115f63c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074953.445912:0:23124:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=sleep, dir=[0x200000401:0x27:0x0](ffff9350115f63c8), file ffff935003061c80, open_flags 80c1, mode 81c0 opened 0 00000080:00200000:1.0:1769074953.445920:0:23124:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x27:0x0](ffff9350115f63c8), intent=open|creat 00000002:00010000:1.0:1769074953.445941:0:23124:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x200000401:0x28:0x0]) in obj [0x200000401:0x27:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1769074953.446000:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 3/0,1 mode: --/CW res: [0x200000401:0x27:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.446007:0:23124:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 3/0,1 mode: --/CW res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00000020:00100000:1.0:1769074953.446019:0:23124:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074953.446028:0:23124:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5880 x1855008913018624/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074953.446036:0:23124:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 3/0,1 mode: --/CW res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00000100:00100000:1.0:1769074953.446102:0:23124:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018624:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074953.446257:0:23124:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001254840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074953.451636:0:23124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018624:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074953.451674:0:23124:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 4/0,1 mode: --/CW res: [0x200000401:0x27:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.451701:0:23124:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 4/0,1 mode: --/PR res: [0x200000401:0x28:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.451710:0:23124:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 4/0,1 mode: --/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.451723:0:23124:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.451741:0:23124:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.451756:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.451764:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074953.451771:0:23124:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074953.451782:0:23124:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff9350062a5880 x1855008913018624/t4294973372(4294973372) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074969 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074953.451795:0:23124:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a5880 x1855008913018624/t4294973372(4294973372) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074969 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074953.451811:0:23124:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074953.451826:0:23124:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.451837:0:23124:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/40 (0 0) 00000080:00200000:1.0:1769074953.451845:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0x27:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074953.451858:0:23124:0:(namei.c:1569:ll_create_it()) VFS Op:name=sleep, dir=[0x200000401:0x27:0x0](ffff9350115f63c8), intent=open|creat 00000080:00200000:1.0:1769074953.451892:0:23124:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000401:0x28:0x0](ffff9350115aba88) 00000080:00200000:1.0:1769074953.451978:0:23124:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x28:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074953.451984:0:23124:0:(namei.c:1535:ll_create_node()) setting l_ast_data to inode [0x200000401:0x28:0x0](ffff9350115aba88) 00000080:00010000:1.0:1769074953.451987:0:23124:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28:0x0](ffff9350115aba88) for lock 0x410246667097e1d8 00000080:00010000:1.0:1769074953.451997:0:23124:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x28:0x0](ffff9350115aba88) 00000080:00200008:1.0:1769074953.452015:0:23124:0:(file.c:1001:ll_file_open()) START file sleep:[0x200000401:0x28:0x0](ffff9350115aba88), flags 100301 00000002:00100000:1.0:1769074953.452036:0:23124:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a5880 x1855008913018624/t4294973372(4294973372) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074969 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074953.452142:0:23124:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1d8 from it ffff93500693d5a0 00010000:00010000:1.0:1769074953.452147:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.452155:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074953.452177:0:23124:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x28:0x0](ffff9350115aba88), inode mode 81c0 mask 40 00000080:00200008:1.0:1769074953.452209:0:23124:0:(file.c:6252:ll_getattr_dentry()) START file sleep:[0x200000401:0x28:0x0](ffff9350115aba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074953.452215:0:23124:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x28:0x0](ffff9350115aba88),name=sleep 00000002:00010000:1.0:1769074953.452235:0:23124:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x28:0x0]) in obj [0x200000401:0x27:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074953.452244:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.452255:0:23124:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074953.452261:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x27:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074953.452266:0:23124:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x28:0x0](ffff9350115aba88) 00000080:00010000:1.0:1769074953.452269:0:23124:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28:0x0](ffff9350115aba88) for lock 0x410246667097e1d8 00000080:00010000:1.0:1769074953.452464:0:23124:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1d8 from it ffffa8e541f33cd8 00010000:00010000:1.0:1769074953.452471:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.452480:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074953.452500:0:23124:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x28:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074953.452512:0:23124:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x28:0x0] 00000008:00010000:1.0:1769074953.452532:0:23124:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074953.452537:0:23124:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c2b0, osc lock ffff935010592f70, flags 0x400001002 00010000:00010000:1.0:1769074953.452596:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1df lrc: 3/1,0 mode: --/PR res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.452606:0:23124:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1e00/0x410246667097e1df lrc: 3/1,0 mode: --/PR res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074953.452643:0:23124:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1769074953.464230:0:23124:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x28:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1769074953.464238:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9350084896d0 (ffff9350053393f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074953.464242:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9350084896d0 (ffff9350053393f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074953.464247:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5c2b0 (ffff935005339bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074953.464251:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9350084896d0 (ffff9350053393f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074953.464253:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5c2b0 (ffff9350053393f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074953.464267:0:23124:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x28:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074953.464622:0:23124:0:(file.c:2648:do_file_write_iter()) START file sleep:[0x200000401:0x28:0x0], ppos: 0, count: 38336 00000080:00200000:1.0:1769074953.464634:0:23124:0:(file.c:1994:ll_file_io_generic()) sleep: write ppos: 0, bytes: 38336 00000080:00200000:1.0:1769074953.464640:0:23124:0:(file.c:1759:ll_io_set_mirror()) sleep: desiginated mirror: 0 00000020:00200000:1.0:1769074953.464647:0:23124:0:(cl_io.c:214:cl_io_rw_init()) header@00000000697d7031[0x4, 1, [0x200000401:0x28:0x0] hash] 00000020:00200000:1.0:1769074953.464648:0:23124:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 38336) 0 0 00000080:00200000:1.0:1769074953.464651:0:23124:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x28:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074953.464663:0:23124:0:(file.c:2072:ll_file_io_generic()) Range lock [0, 9] 00020000:00200000:1.0:1769074953.464669:0:23124:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 38336) 38336, 38336 00020000:00200000:1.0:1769074953.464672:0:23124:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074953.464685:0:23124:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 9] 00000008:00010000:1.0:1769074953.464699:0:23124:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074953.464704:0:23124:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c2b0, osc lock ffff935010592f70, flags 0x0 00010000:00010000:1.0:1769074953.464724:0:23124:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935006fa6c00 type 11 mode 2 res 10737419264/11 (0 40959) 00010000:00010000:1.0:1769074953.465368:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 3/0,1 mode: --/PW res: [0x280000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.465379:0:23124:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 3/0,1 mode: --/PW res: [0x280000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.465387:0:23124:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 3/0,1 mode: --/PW res: [0x280000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074953.465409:0:23124:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018880:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074953.465958:0:23124:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001254b40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074953.485594:0:23124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913018880:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074953.485631:0:23124:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.485651:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074953.485667:0:23124:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.485679:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.485692:0:23124:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074953.485727:0:23124:0:(vvp_io.c:1312:vvp_io_write_start()) sleep: write [0, 38336) append 0 append lockpos 0 00000080:00200000:1.0:1769074953.485748:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.485814:0:23124:0:(rw26.c:912:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1769074953.485830:0:23124:0:(rw26.c:969:ll_write_end()) page@0000000018793bc2[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.485834:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @000000002bfbf79c 17ffffc0001001 3:0 ffff935007e08438 0 lru 00000080:00200000:1.0:1769074953.485841:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@000000000d595765 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.485843:0:23124:0:(rw26.c:969:ll_write_end()) end page@0000000018793bc2 00000080:00200000:1.0:1769074953.485844:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1769074953.485848:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.485873:0:23124:0:(rw26.c:912:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:1.0:1769074953.485877:0:23124:0:(rw26.c:969:ll_write_end()) page@00000000888f21f8[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.485880:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @00000000f1cc3f4b 17ffffc0001001 3:0 ffff935007e086c0 1 lru 00000080:00200000:1.0:1769074953.485885:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@00000000b43ce00d 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.485887:0:23124:0:(rw26.c:969:ll_write_end()) end page@00000000888f21f8 00000080:00200000:1.0:1769074953.485888:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 2. 00000080:00200000:1.0:1769074953.485890:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.485905:0:23124:0:(rw26.c:912:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:1.0:1769074953.485908:0:23124:0:(rw26.c:969:ll_write_end()) page@0000000067071cbd[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.485911:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @00000000477dba4b 17ffffc0001001 3:0 ffff935007e08af8 2 lru 00000080:00200000:1.0:1769074953.485915:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@0000000023a08f6c 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.485917:0:23124:0:(rw26.c:969:ll_write_end()) end page@0000000067071cbd 00000080:00200000:1.0:1769074953.485918:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 3. 00000080:00200000:1.0:1769074953.485920:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.485946:0:23124:0:(rw26.c:912:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:1.0:1769074953.485951:0:23124:0:(rw26.c:969:ll_write_end()) page@0000000029c5d06a[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.485953:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @000000002a36a743 17ffffc0001021 2:0 ffff935007e08ca8 3 lru 00000080:00200000:1.0:1769074953.485959:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@000000007da82ca9 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.485961:0:23124:0:(rw26.c:969:ll_write_end()) end page@0000000029c5d06a 00000080:00200000:1.0:1769074953.485962:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 4. 00000080:00200000:1.0:1769074953.485963:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.485979:0:23124:0:(rw26.c:912:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:1.0:1769074953.485983:0:23124:0:(rw26.c:969:ll_write_end()) page@000000000e0bb216[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.485986:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @00000000598de579 17ffffc0001001 3:0 ffff935007e08a20 4 lru 00000080:00200000:1.0:1769074953.485992:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@00000000afd7cfa4 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.485994:0:23124:0:(rw26.c:969:ll_write_end()) end page@000000000e0bb216 00000080:00200000:1.0:1769074953.485995:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 5. 00000080:00200000:1.0:1769074953.485997:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.486008:0:23124:0:(rw26.c:912:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:1.0:1769074953.486012:0:23124:0:(rw26.c:969:ll_write_end()) page@0000000085e0ac0e[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.486014:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @0000000053344b34 17ffffc0001001 3:0 ffff935007e081b0 5 lru 00000080:00200000:1.0:1769074953.486019:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@000000008bd5e72a 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.486021:0:23124:0:(rw26.c:969:ll_write_end()) end page@0000000085e0ac0e 00000080:00200000:1.0:1769074953.486022:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 6. 00000080:00200000:1.0:1769074953.486024:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.486036:0:23124:0:(rw26.c:912:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:1.0:1769074953.486040:0:23124:0:(rw26.c:969:ll_write_end()) page@0000000040d6d867[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.486043:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @000000007a5e5534 17ffffc0001001 3:0 ffff935007e08870 6 lru 00000080:00200000:1.0:1769074953.486070:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@00000000b4b72bc5 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.486073:0:23124:0:(rw26.c:969:ll_write_end()) end page@0000000040d6d867 00000080:00200000:1.0:1769074953.486074:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 7. 00000080:00200000:1.0:1769074953.486076:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.486093:0:23124:0:(rw26.c:912:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:1.0:1769074953.486097:0:23124:0:(rw26.c:969:ll_write_end()) page@00000000410f028f[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.486100:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @00000000505536a0 17ffffc0001001 3:0 ffff935007e08798 7 lru 00000080:00200000:1.0:1769074953.486106:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@000000003da6bdc5 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.486108:0:23124:0:(rw26.c:969:ll_write_end()) end page@00000000410f028f 00000080:00200000:1.0:1769074953.486109:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 8. 00000080:00200000:1.0:1769074953.486111:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:1.0:1769074953.486126:0:23124:0:(rw26.c:912:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:1.0:1769074953.486131:0:23124:0:(rw26.c:969:ll_write_end()) page@00000000ec99b1f4[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.486133:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @00000000eb6e35a3 17ffffc0001001 3:0 ffff935007e08360 8 lru 00000080:00200000:1.0:1769074953.486139:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@00000000cb53a965 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.486141:0:23124:0:(rw26.c:969:ll_write_end()) end page@00000000ec99b1f4 00000080:00200000:1.0:1769074953.486142:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 9. 00000080:00200000:1.0:1769074953.486143:0:23124:0:(rw26.c:729:ll_write_begin()) Writing 9 of 0 to 1472 bytes 00000080:00200000:1.0:1769074953.486178:0:23124:0:(rw26.c:912:ll_write_end()) pos 36864, len 1472, copied 1472 00000080:00200000:1.0:1769074953.486183:0:23124:0:(rw26.c:969:ll_write_end()) page@00000000a04dc877[3 00000000c35147a8 2 1 000000006bf70109] 00000080:00200000:1.0:1769074953.486186:0:23124:0:(rw26.c:969:ll_write_end()) vmpage @00000000a39ac8d0 17ffffc0001001 3:0 ffff935007e08510 9 lru 00000080:00200000:1.0:1769074953.486192:0:23124:0:(rw26.c:969:ll_write_end()) osc-page@00000000711e1c08 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000065e64005 000000002429a1c5 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074953.486194:0:23124:0:(rw26.c:969:ll_write_end()) end page@00000000a04dc877 00000080:00200000:1.0:1769074953.486195:0:23124:0:(rw26.c:969:ll_write_end()) queued page: 10. 00000080:00200000:1.0:1769074953.486199:0:23124:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 10, from 0, to 1472 prio 0 00000008:00010000:1.0:1769074953.486224:0:23124:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000931f19d9 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074953.486253:0:23124:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9350115abc38, count 10, dirtied 10 00000080:00200000:1.0:1769074953.486257:0:23124:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 10 pages 38336 bytes, tot: 38336 00000080:00200000:1.0:1769074953.486260:0:23124:0:(vvp_io.c:1390:vvp_io_write_start()) sleep: write bytes 0, result: 38336 00000020:00010000:1.0:1769074953.486276:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074953.486280:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074953.486283:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5c2b0 (ffff935005339bd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074953.486288:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.486298:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074953.486315:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074953.486318:0:23124:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5c2b0 (ffff9350053393f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074953.486327:0:23124:0:(vvp_io.c:518:vvp_io_advance()) advancing 38336 bytes 00000080:00200000:1.0:1769074953.486335:0:23124:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 9] 00000080:00200000:1.0:1769074953.486344:0:23124:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x28:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074953.486348:0:23124:0:(file.c:2142:ll_file_io_generic()) sleep: 2 io complete with rc: 0, result: 38336, restart: 0 00000080:00200000:1.0:1769074953.486355:0:23124:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x28:0x0] updating i_size 38336 i_blocks 1 00000080:00200000:1.0:1769074953.486361:0:23124:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 38336 00000080:00200008:1.0:1769074953.486405:0:23124:0:(llite_lib.c:2314:ll_setattr_raw()) START file sleep:[0x200000401:0x28:0x0](ffff9350115aba88) current size 38336, valid attrs 101f0, mode ffff, uid 50732160, gid 131072, new size 18446624570590238720, atime 1752421453.000000000, mtime 1674053537.000000000, ctime 1769074953.000000000 00010000:00010000:1.0:1769074953.486438:0:23124:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074953.486471:0:23124:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x28:0x0]: losing layout lock 00010000:00010000:1.0:1769074953.486525:0:23124:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.486534:0:23124:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074953.486536:0:23124:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1d8 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97c29 expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074953.486579:0:23124:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074953.486589:0:23124:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6d80 x1855008913019264/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074953.486615:0:23124:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913019264:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074953.486700:0:23124:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001254b40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074953.496625:0:23124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913019264:192.168.204.121@tcp:36:cp.0 00000080:00200000:1.0:1769074953.496680:0:23124:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x28:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:1.0:1769074953.496717:0:23124:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0x28:0x0](ffff9350115aba88) 00000002:00010000:1.0:1769074953.496732:0:23124:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x28:0x0]) in obj [0x200000401:0x28:0x0], intent: layout flags 0 00010000:00010000:1.0:1769074953.496795:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: --/CR res: [0x200000401:0x28:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.496806:0:23124:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: --/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00010000:00010000:1.0:1769074953.496818:0:23124:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: --/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 23124 initiator: MDT0 00000100:00100000:1.0:1769074953.496841:0:23124:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913019392:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074953.496907:0:23124:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001254b40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074953.500135:0:23124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23124:1855008913019392:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074953.500189:0:23124:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.500217:0:23124:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1769074953.500234:0:23124:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9350062a5c00 x1855008913019392/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 376/400 e 0 to 0 dl 1769074969 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074953.500247:0:23124:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000002:00010000:1.0:1769074953.500262:0:23124:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.500278:0:23124:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 16 res 8589935617/40 (0 0) 00000080:00200000:1.0:1769074953.500286:0:23124:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0x28:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1769074953.500307:0:23124:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28:0x0](ffff9350115aba88) for lock 0x410246667097e1ed 00000080:00010000:1.0:1769074953.500316:0:23124:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1ed from it ffffa8e541f33a78 00010000:00010000:1.0:1769074953.500321:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.500331:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.500355:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.500368:0:23124:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1769074953.500376:0:23124:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000401:0x28:0x0](0000000030fcc645) being reconfigured ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074953.500432:0:23124:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x28:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:1.0:1769074953.500438:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074953.500444:0:23124:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00020000:00200000:1.0:1769074953.500475:0:23124:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074953.500492:0:23124:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1769074953.500509:0:23124:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074953.500514:0:23124:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c2b0, osc lock ffff935010592f70, flags 0x0 00010000:00010000:1.0:1769074953.500533:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.500543:0:23124:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.500561:0:23124:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.500569:0:23124:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074953.500575:0:23124:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074953.500666:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074953.500686:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913019520:192.168.204.121@tcp:2:cp.0 00000100:00100000:1.0:1769074953.511904:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913019520:192.168.204.121@tcp:2:cp.0 00010000:00010000:1.0:1769074955.441708:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074956.465390:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074957.300274:0:37:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074957.300294:0:37:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074957.300320:0:37:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074957.300339:0:37:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x23:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074957.300345:0:37:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074957.300353:0:37:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x23:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074957.300360:0:37:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074957.300364:0:37:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074957.300369:0:37:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074957.300375:0:37:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x28:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074957.300379:0:37:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1769074957.300411:0:37:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 00000000931f19d9 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074957.300433:0:37:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 00000000931f19d9 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074957.300505:0:37:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074957.300627:0:37:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9350062a5180 - object 0x280000400:11 offset 0<>38336 00000100:00100000:1.0:1769074957.300699:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913021568:192.168.204.121@tcp:4:cp.0 00000100:00100000:1.0:1769074957.300820:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074957.300828:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913021696:192.168.204.121@tcp:16:kworker.0 00000100:00100000:1.0:1769074957.313790:0:2362:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9350067ee000: committing for last_committed 4294967324 gen 2 00000100:00100000:1.0:1769074957.313844:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913021696:192.168.204.121@tcp:16:kworker.0 00000080:00200000:1.0:1769074957.313887:0:37:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x28:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1769074957.325636:0:2362:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 00000000931f19d9 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074957.325703:0:2362:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 00000000931f19d9 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa1600/0x410246667097e1e6 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97c3e expref: -99 pid: 23124 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074957.325741:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913021568:192.168.204.121@tcp:4:cp.0 00010000:00010000:1.0:1769074959.538497:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074960.431840:0:23700:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074960.431872:0:23700:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074960.431893:0:23700:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.431911:0:23700:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.431919:0:23700:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074960.431926:0:23700:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074960.431930:0:23700:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e1bc 00000080:00010000:1.0:1769074960.431937:0:23700:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1bc from it ffffa8e541ea3be0 00010000:00010000:1.0:1769074960.431941:0:23700:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.431962:0:23700:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.431984:0:23700:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074960.432002:0:23700:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d14b.sanityn, dir=[0x200000007:0x1:0x0](ffff935011534b08), flags=1536 00000080:00200000:1.0:1769074960.432006:0:23700:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074960.435347:0:23700:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074960.435369:0:23700:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.435388:0:23700:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.435396:0:23700:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074960.435407:0:23700:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074960.435411:0:23700:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e1bc 00000080:00010000:1.0:1769074960.435419:0:23700:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1bc from it ffffa8e541ea3ce0 00010000:00010000:1.0:1769074960.435423:0:23700:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.435431:0:23700:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.435449:0:23700:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 3 00000080:00200000:1.0:1769074960.435463:0:23700:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074960.435470:0:23700:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074960.435475:0:23700:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.435482:0:23700:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.435488:0:23700:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074960.435491:0:23700:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074960.435701:0:23700:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e1bc 00000080:00010000:1.0:1769074960.435707:0:23700:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e1bc from it ffffa8e541ea3dc8 00010000:00010000:1.0:1769074960.435710:0:23700:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.435718:0:23700:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.435727:0:23700:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 3 00000080:00200000:1.0:1769074960.435733:0:23700:0:(namei.c:2175:do_mkdir()) VFS Op:name=d14b.sanityn, dir=[0x200000007:0x1:0x0](ffff935011534b08) 00010000:00010000:1.0:1769074960.435768:0:23700:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.435834:0:23700:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.435841:0:23700:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074960.435844:0:23700:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e1bc lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x48bf02eaaaf97bc7 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074960.435864:0:23700:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074960.435878:0:23700:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5500 x1855008913022720/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 560/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074960.435906:0:23700:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:23700:1855008913022720:192.168.204.121@tcp:36:mkdir.0 00000100:00100000:1.0:1769074960.436459:0:23700:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00010000:00010000:1.0:1769074960.442700:0:14429:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074960.442706:0:14429:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935002ea5e00/0x410246667097e202 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x48bf02eaaaf97c99 expref: -99 pid: 23126 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.442713:0:14429:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935002ea5e00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074960.442718:0:14429:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935002ea5e00/0x410246667097e202 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97c99 expref: -99 pid: 23126 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.442723:0:14429:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935002ea5e00/0x410246667097e202 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97c99 expref: -99 pid: 23126 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.442729:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935002ea5e00/0x410246667097e202 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x48bf02eaaaf97c99 expref: -99 pid: 23126 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.442788:0:14429:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935002ea5e00/0x410246667097e202 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97c99 expref: -99 pid: 23126 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.442793:0:14429:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074960.442822:0:14429:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935002ea5e00/0x410246667097e202 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97c99 expref: -99 pid: 23126 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.442827:0:14429:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935002ea5e00/0x410246667097e202 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97c99 expref: -99 pid: 23126 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.442837:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074960.442838:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074960.442839:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074960.442846:0:2363:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074960.442859:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913022848:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074960.450438:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913022848:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074960.452812:0:23700:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job mkdir:7afd476b-94ee-4c94-b080-d15824f3ae56:23700:1855008913022720:192.168.204.121@tcp:36:mkdir.0 00000100:00100000:1.0:1769074960.627764:0:23704:0:(client.c:2690:ptlrpc_set_wait()) set ffff935001663780 going to sleep for 11 seconds 00000080:00200000:1.0:1769074960.651446:0:23704:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x2a:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1769074960.651459:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074960.651463:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074960.651470:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935004e50b30 (ffff9350053391b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074960.651475:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074960.651478:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935004e50b30 (ffff935005339ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074960.651641:0:23704:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2a:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074960.651973:0:23704:0:(file.c:2648:do_file_write_iter()) START file sleep:[0x200000401:0x2a:0x0], ppos: 0, count: 38336 00000080:00200000:1.0:1769074960.651985:0:23704:0:(file.c:1994:ll_file_io_generic()) sleep: write ppos: 0, bytes: 38336 00000080:00200000:1.0:1769074960.651993:0:23704:0:(file.c:1759:ll_io_set_mirror()) sleep: desiginated mirror: 0 00000020:00200000:1.0:1769074960.652000:0:23704:0:(cl_io.c:214:cl_io_rw_init()) header@00000000b7a0e367[0x4, 1, [0x200000401:0x2a:0x0] hash] 00000020:00200000:1.0:1769074960.652003:0:23704:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 38336) 0 0 00000080:00200000:1.0:1769074960.652007:0:23704:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074960.652021:0:23704:0:(file.c:2072:ll_file_io_generic()) Range lock [0, 9] 00020000:00200000:1.0:1769074960.652030:0:23704:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 38336) 38336, 38336 00020000:00200000:1.0:1769074960.652034:0:23704:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074960.652204:0:23704:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 9] 00000008:00010000:1.0:1769074960.652220:0:23704:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074960.652225:0:23704:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c730, osc lock ffff935010593c80, flags 0x0 00010000:00010000:1.0:1769074960.652248:0:23704:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d4c00 type 11 mode 2 res 9663677440/11 (0 40959) 00010000:00010000:1.0:1769074960.652635:0:23704:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 3/0,1 mode: --/PW res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.652649:0:23704:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 3/0,1 mode: --/PW res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.652663:0:23704:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 3/0,1 mode: --/PW res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074960.654665:0:23704:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913023872:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074960.655466:0:23704:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.670379:0:23704:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913023872:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074960.670415:0:23704:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,1 mode: --/PW res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.670429:0:23704:0:(ldlm_request.c:274:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,1 mode: --/PW res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.670438:0:23704:0:(ldlm_request.c:303:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,1 mode: --/PW res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.671839:0:14430:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93500316b800/0x410246667097e23a lrc: 2/0,0 mode: --/PR res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97d5d expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.671847:0:14430:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074960.673765:0:14430:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93500316b800/0x410246667097e23a lrc: 1/0,0 mode: --/PR res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97d5d expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.673785:0:14430:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff93500316b800/0x410246667097e23a lrc: 0/0,0 mode: --/PR res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97d5d expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.673817:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074960.673820:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074960.673823:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074960.676476:0:23704:0:(ldlm_request.c:186:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.677300:0:23704:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.677322:0:23704:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074960.677352:0:23704:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.677364:0:23704:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.677378:0:23704:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074960.677416:0:23704:0:(vvp_io.c:1312:vvp_io_write_start()) sleep: write [0, 38336) append 0 append lockpos 0 00000080:00200000:1.0:1769074960.677440:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677515:0:23704:0:(rw26.c:912:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677530:0:23704:0:(rw26.c:969:ll_write_end()) page@00000000acd42cfc[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677534:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @00000000645d46eb 17ffffc0001001 3:0 ffff935007e08bd0 0 lru 00000080:00200000:1.0:1769074960.677541:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@00000000b8641563 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677543:0:23704:0:(rw26.c:969:ll_write_end()) end page@00000000acd42cfc 00000080:00200000:1.0:1769074960.677544:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1769074960.677548:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677573:0:23704:0:(rw26.c:912:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677578:0:23704:0:(rw26.c:969:ll_write_end()) page@00000000947a0738[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677581:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @00000000d8792971 17ffffc0001001 3:0 ffff935007e08d80 1 lru 00000080:00200000:1.0:1769074960.677586:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@000000005a76b5ab 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677588:0:23704:0:(rw26.c:969:ll_write_end()) end page@00000000947a0738 00000080:00200000:1.0:1769074960.677589:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 2. 00000080:00200000:1.0:1769074960.677591:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677606:0:23704:0:(rw26.c:912:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677610:0:23704:0:(rw26.c:969:ll_write_end()) page@00000000cf1e9267[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677613:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @000000001400b7c0 17ffffc0001001 3:0 ffff935007e08000 2 lru 00000080:00200000:1.0:1769074960.677618:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@00000000918cd7cd 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677620:0:23704:0:(rw26.c:969:ll_write_end()) end page@00000000cf1e9267 00000080:00200000:1.0:1769074960.677621:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 3. 00000080:00200000:1.0:1769074960.677623:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677648:0:23704:0:(rw26.c:912:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677651:0:23704:0:(rw26.c:969:ll_write_end()) page@00000000a32dc9a3[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677654:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @000000002601311e 17ffffc0001021 2:0 ffff935007e085e8 3 lru 00000080:00200000:1.0:1769074960.677658:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@00000000c5835da9 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677660:0:23704:0:(rw26.c:969:ll_write_end()) end page@00000000a32dc9a3 00000080:00200000:1.0:1769074960.677661:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 4. 00000080:00200000:1.0:1769074960.677663:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677677:0:23704:0:(rw26.c:912:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677680:0:23704:0:(rw26.c:969:ll_write_end()) page@0000000008227474[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677683:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @00000000821da543 17ffffc0001001 3:0 ffff935007e08288 4 lru 00000080:00200000:1.0:1769074960.677687:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@000000000023c0d4 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677689:0:23704:0:(rw26.c:969:ll_write_end()) end page@0000000008227474 00000080:00200000:1.0:1769074960.677690:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 5. 00000080:00200000:1.0:1769074960.677691:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677713:0:23704:0:(rw26.c:912:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677717:0:23704:0:(rw26.c:969:ll_write_end()) page@0000000046dccfe7[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677719:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @000000002e339663 17ffffc0001001 3:0 ffff935007e080d8 5 lru 00000080:00200000:1.0:1769074960.677723:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@00000000c7905d78 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677725:0:23704:0:(rw26.c:969:ll_write_end()) end page@0000000046dccfe7 00000080:00200000:1.0:1769074960.677726:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 6. 00000080:00200000:1.0:1769074960.677728:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677741:0:23704:0:(rw26.c:912:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677745:0:23704:0:(rw26.c:969:ll_write_end()) page@00000000b508a0b7[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677747:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @000000008ccfb979 17ffffc0001001 3:0 ffff935007e08e58 6 lru 00000080:00200000:1.0:1769074960.677751:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@00000000c42cb2c7 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677753:0:23704:0:(rw26.c:969:ll_write_end()) end page@00000000b508a0b7 00000080:00200000:1.0:1769074960.677755:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 7. 00000080:00200000:1.0:1769074960.677757:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677770:0:23704:0:(rw26.c:912:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677773:0:23704:0:(rw26.c:969:ll_write_end()) page@000000008d7e509c[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677776:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @00000000f274a54e 17ffffc0001001 3:0 ffff935007e08948 7 lru 00000080:00200000:1.0:1769074960.677781:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@00000000982271cd 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677783:0:23704:0:(rw26.c:969:ll_write_end()) end page@000000008d7e509c 00000080:00200000:1.0:1769074960.677784:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 8. 00000080:00200000:1.0:1769074960.677786:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:1.0:1769074960.677803:0:23704:0:(rw26.c:912:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:1.0:1769074960.677807:0:23704:0:(rw26.c:969:ll_write_end()) page@00000000551e6400[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677810:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @00000000239c1cc4 17ffffc0001001 3:0 ffff9350109da870 8 lru 00000080:00200000:1.0:1769074960.677815:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@00000000f5106a10 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677818:0:23704:0:(rw26.c:969:ll_write_end()) end page@00000000551e6400 00000080:00200000:1.0:1769074960.677819:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 9. 00000080:00200000:1.0:1769074960.677820:0:23704:0:(rw26.c:729:ll_write_begin()) Writing 9 of 0 to 1472 bytes 00000080:00200000:1.0:1769074960.677843:0:23704:0:(rw26.c:912:ll_write_end()) pos 36864, len 1472, copied 1472 00000080:00200000:1.0:1769074960.677847:0:23704:0:(rw26.c:969:ll_write_end()) page@0000000079969d24[3 0000000042110154 2 1 00000000b546752c] 00000080:00200000:1.0:1769074960.677850:0:23704:0:(rw26.c:969:ll_write_end()) vmpage @00000000b18c1697 17ffffc0001001 3:0 ffff9350109da1b0 9 lru 00000080:00200000:1.0:1769074960.677854:0:23704:0:(rw26.c:969:ll_write_end()) osc-page@000000006755919c 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000001c2e547 00000000e90a95d0 > 3< 0 0 > 4< 0 0 8 19136512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074960.677856:0:23704:0:(rw26.c:969:ll_write_end()) end page@0000000079969d24 00000080:00200000:1.0:1769074960.677857:0:23704:0:(rw26.c:969:ll_write_end()) queued page: 10. 00000080:00200000:1.0:1769074960.677860:0:23704:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 10, from 0, to 1472 prio 0 00000008:00010000:1.0:1769074960.677882:0:23704:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000931f19d9 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074960.677910:0:23704:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9350115f4478, count 10, dirtied 10 00000080:00200000:1.0:1769074960.677916:0:23704:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 10 pages 38336 bytes, tot: 38336 00000080:00200000:1.0:1769074960.677919:0:23704:0:(vvp_io.c:1390:vvp_io_write_start()) sleep: write bytes 0, result: 38336 00000020:00010000:1.0:1769074960.677937:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074960.677941:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074960.677944:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5c730 (ffff9350053391b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074960.677949:0:23704:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.677959:0:23704:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074960.677977:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074960.677981:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5c730 (ffff935005339ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074960.677990:0:23704:0:(vvp_io.c:518:vvp_io_advance()) advancing 38336 bytes 00000080:00200000:1.0:1769074960.677997:0:23704:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 9] 00000080:00200000:1.0:1769074960.678006:0:23704:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074960.678010:0:23704:0:(file.c:2142:ll_file_io_generic()) sleep: 2 io complete with rc: 0, result: 38336, restart: 0 00000080:00200000:1.0:1769074960.678018:0:23704:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x2a:0x0] updating i_size 38336 i_blocks 1 00000080:00200000:1.0:1769074960.678024:0:23704:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 38336 00000080:00200008:1.0:1769074960.678111:0:23704:0:(llite_lib.c:2314:ll_setattr_raw()) START file sleep:[0x200000401:0x2a:0x0](ffff9350115f42c8) current size 38336, valid attrs 101f0, mode ffff, uid 58532608, gid 131072, new size 18446624570598046848, atime 1752421453.000000000, mtime 1674053537.000000000, ctime 1769074960.000000000 00010000:00010000:1.0:1769074960.678143:0:23704:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ac00/0x410246667097e233 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97d4f expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074960.678164:0:23704:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x2a:0x0]: losing layout lock 00010000:00010000:1.0:1769074960.678217:0:23704:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ac00/0x410246667097e233 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97d4f expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074960.678225:0:23704:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074960.678229:0:23704:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316ac00/0x410246667097e233 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97d4f expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074960.678256:0:23704:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074960.678270:0:23704:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a5c00 x1855008913024256/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074960.678297:0:23704:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913024256:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074960.678415:0:23704:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.689529:0:23704:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913024256:192.168.204.121@tcp:36:cp.0 00000080:00200000:1.0:1769074960.689585:0:23704:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:1.0:1769074960.689611:0:23704:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0x2a:0x0](ffff9350115f42c8) 00000002:00010000:1.0:1769074960.689623:0:23704:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a:0x0]) in obj [0x200000401:0x2a:0x0], intent: layout flags 0 00010000:00010000:1.0:1769074960.689688:0:23704:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 23704 initiator: MDT0 00010000:00010000:1.0:1769074960.689699:0:23704:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 23704 initiator: MDT0 00010000:00010000:1.0:1769074960.689711:0:23704:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 23704 initiator: MDT0 00000100:00100000:1.0:1769074960.689732:0:23704:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913024384:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074960.689807:0:23704:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.698974:0:23704:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913024384:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074960.699015:0:23704:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074960.699033:0:23704:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1769074960.699105:0:23704:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9350062a5880 x1855008913024384/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 376/400 e 0 to 0 dl 1769074976 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074960.699117:0:23704:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00000002:00010000:1.0:1769074960.699131:0:23704:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074960.699146:0:23704:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 16 res 8589935617/42 (0 0) 00000080:00200000:1.0:1769074960.699154:0:23704:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0x2a:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1769074960.699174:0:23704:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a:0x0](ffff9350115f42c8) for lock 0x410246667097e248 00000080:00010000:1.0:1769074960.699183:0:23704:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e248 from it ffffa8e541eb3a78 00010000:00010000:1.0:1769074960.699187:0:23704:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074960.699197:0:23704:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074960.699219:0:23704:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074960.699232:0:23704:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1769074960.699240:0:23704:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000401:0x2a:0x0](000000000c99b516) being reconfigured ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074960.699287:0:23704:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x2a:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:1.0:1769074960.699293:0:23704:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074960.699300:0:23704:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00020000:00200000:1.0:1769074960.699322:0:23704:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074960.699349:0:23704:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1769074960.699365:0:23704:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074960.699369:0:23704:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5ce30, osc lock ffff935010593c80, flags 0x0 00010000:00010000:1.0:1769074960.699387:0:23704:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.699397:0:23704:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.699415:0:23704:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.699423:0:23704:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.699429:0:23704:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074960.703694:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074960.703700:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074960.703717:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5ce30 (ffff9350053391b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074960.703731:0:23704:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074960.703747:0:23704:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074960.703767:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074960.703770:0:23704:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5ce30 (ffff935005339ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074960.703792:0:23704:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:1.0:1769074960.703811:0:23704:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff93501032d400 uarg=0000000000000000 00000002:00000080:1.0:1769074960.703823:0:23704:0:(mdc_request.c:2279:mdc_iocontrol()) lustre-MDT0000-mdc-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff93501032d400 uarg=0000000000000000 00000020:00100000:1.0:1769074960.703860:0:23704:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:1.0:1769074960.703870:0:23704:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4380 x1855008913024768/t0(0) o55->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074960.703899:0:23704:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913024768:192.168.204.121@tcp:55:cp.0 00000100:00100000:1.0:1769074960.704004:0:23704:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.708518:0:23704:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913024768:192.168.204.121@tcp:55:cp.0 00000080:00200008:1.0:1769074960.708569:0:23704:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file sleep:[0x200000401:0x2a:0x0](ffff9350115f42c8) current size 38336, valid attrs 101f0, mode ffff, uid 58532608, gid 131072, new size 18446624570598046848, atime 1752421453.000000000, mtime 1674053537.000000000, ctime 1769074960.000000000, rc 0 00000020:00100000:1.0:1769074960.708723:0:23704:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074960.708732:0:23704:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4380 x1855008913025024/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 544/440 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074960.708757:0:23704:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913025024:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074960.708851:0:23704:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.713760:0:23704:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913025024:192.168.204.121@tcp:36:cp.0 00000080:00200008:1.0:1769074960.713836:0:23704:0:(file.c:424:ll_file_release()) START file sleep:[0x200000401:0x2a:0x0](ffff9350115f42c8), flags 100001 00000002:00100000:1.0:1769074960.713905:0:23704:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff935002a62a00 x1855008913023616/t4294973379(4294973379) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074976 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074960.713917:0:23704:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074960.713922:0:23704:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008913025280/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074960.713946:0:23704:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913025280:192.168.204.121@tcp:35:cp.0 00000100:00100000:1.0:1769074960.714031:0:23704:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.719974:0:23704:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:23704:1855008913025280:192.168.204.121@tcp:35:cp.0 00000002:00100000:1.0:1769074960.720014:0:23704:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff935002a62a00 x1855008913023616/t4294973379(4294973379) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074976 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074960.728483:0:23706:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8),name=/ 00000002:00010000:1.0:1769074960.728522:0:23706:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074960.728544:0:23706:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 30 res 8589934599/1 (0 0) 00010000:00010000:1.0:1769074960.728610:0:23706:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23706 initiator: MDT0 00010000:00010000:1.0:1769074960.728620:0:23706:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23706 initiator: MDT0 00010000:00010000:1.0:1769074960.728630:0:23706:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23706 initiator: MDT0 00000100:00100000:1.0:1769074960.728660:0:23706:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job truncate:84af2899-c956-435d-9e3c-5d5fbc3c9246:23706:1855008913025408:192.168.204.121@tcp:101:truncate.0 00000100:00100000:1.0:1769074960.728771:0:23706:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.735815:0:23706:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job truncate:84af2899-c956-435d-9e3c-5d5fbc3c9246:23706:1855008913025408:192.168.204.121@tcp:101:truncate.0 00010000:00010000:1.0:1769074960.735850:0:23706:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.735864:0:23706:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.735877:0:23706:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.735889:0:23706:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.735902:0:23706:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.735909:0:23706:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.735917:0:23706:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074960.735929:0:23706:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a5f80 x1855008913025408/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074976 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074960.735944:0:23706:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.735962:0:23706:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074960.735971:0:23706:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074960.736011:0:23706:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 12800 00000080:00010000:1.0:1769074960.736020:0:23706:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) for lock 0x410246667097e24f 00000080:00010000:1.0:1769074960.736026:0:23706:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff93501158c2c8) 00000080:00010000:1.0:1769074960.736031:0:23706:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e24f from it ffffa8e5416c3c08 00010000:00010000:1.0:1769074960.736072:0:23706:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.736082:0:23706:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.736115:0:23706:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff93501158c2c8), inode mode 41ff mask 1 00000080:00200000:1.0:1769074960.736141:0:23706:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d14b.sanityn, dir=[0x200000007:0x1:0x0](ffff93501158c2c8), flags=25 00000080:00200000:1.0:1769074960.736146:0:23706:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d14b.sanityn, dir=[0x200000007:0x1:0x0](ffff93501158c2c8), intent=0 00000002:00010000:1.0:1769074960.736168:0:23706:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d14b.sanityn,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074960.736221:0:23706:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 23706 initiator: MDT0 00010000:00010000:1.0:1769074960.736228:0:23706:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 23706 initiator: MDT0 00010000:00010000:1.0:1769074960.736238:0:23706:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 23706 initiator: MDT0 00000100:00100000:1.0:1769074960.736262:0:23706:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job truncate:84af2899-c956-435d-9e3c-5d5fbc3c9246:23706:1855008913025536:192.168.204.121@tcp:101:truncate.0 00000100:00100000:1.0:1769074960.744330:0:23706:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.744401:0:23706:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job truncate:84af2899-c956-435d-9e3c-5d5fbc3c9246:23706:1855008913025536:192.168.204.121@tcp:101:truncate.0 00010000:00010000:1.0:1769074960.744426:0:23706:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744452:0:23706:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744461:0:23706:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744474:0:23706:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744485:0:23706:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744692:0:23706:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744718:0:23706:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744726:0:23706:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074960.744738:0:23706:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff9350062a5f80 x1855008913025536/t0(0) o101->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074976 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074960.744752:0:23706:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744763:0:23706:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff935008499400 type 13 mode 4 res 8589935617/41 (0 0) 00000080:00200000:1.0:1769074960.744771:0:23706:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074960.744805:0:23706:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000401:0x29:0x0](ffff9350115ae3c8) 00000080:00200000:1.0:1769074960.744812:0:23706:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x29:0x0], updating i_size 512 00000080:00010000:1.0:1769074960.744819:0:23706:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x29:0x0](ffff9350115ae3c8) for lock 0x410246667097e256 00000080:00010000:1.0:1769074960.744831:0:23706:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x29:0x0](ffff9350115ae3c8) 00000080:00010000:1.0:1769074960.744836:0:23706:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e256 from it ffffa8e5416c3ad8 00010000:00010000:1.0:1769074960.744839:0:23706:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074960.744848:0:23706:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa1400/0x410246667097e256 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97daa expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074960.744886:0:23706:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x29:0x0](ffff9350115ae3c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074960.744903:0:23706:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=sleep, dir=[0x200000401:0x29:0x0](ffff9350115ae3c8), flags=9 00000080:00200000:1.0:1769074960.744908:0:23706:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x29:0x0](ffff9350115ae3c8), intent=getattr 00000002:00010000:1.0:1769074960.744927:0:23706:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x0:0x0:0x0]) in obj [0x200000401:0x29:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074960.744973:0:23706:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0600/0x410246667097e264 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x29:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23706 initiator: MDT0 00010000:00010000:1.0:1769074960.744980:0:23706:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0600/0x410246667097e264 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x29:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23706 initiator: MDT0 00010000:00010000:1.0:1769074960.744990:0:23706:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0600/0x410246667097e264 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x29:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23706 initiator: MDT0 00000100:00100000:1.0:1769074960.745010:0:23706:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job truncate:84af2899-c956-435d-9e3c-5d5fbc3c9246:23706:1855008913025920:192.168.204.121@tcp:101:truncate.0 00000100:00100000:1.0:1769074960.745431:0:23706:0:(client.c:2690:ptlrpc_set_wait()) set ffff9350084dc540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074960.759963:0:23706:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job truncate:84af2899-c956-435d-9e3c-5d5fbc3c9246:23706:1855008913025920:192.168.204.121@tcp:101:truncate.0 00010000:00010000:1.0:1769074960.759988:0:23706:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0600/0x410246667097e264 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x29:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97de9 expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074960.776880:0:23706:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job truncate:84af2899-c956-435d-9e3c-5d5fbc3c9246:23706:1855008913026304:192.168.204.121@tcp:36:truncate.0 00000080:00200008:1.0:1769074960.776920:0:23706:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file sleep:[0x200000401:0x2a:0x0](ffff9350116321c8) current size 0, valid attrs 168, mode ffff, uid -1163628244, gid -1, new size 60, atime 1769074960.000000000, mtime 1769074960.000000000, ctime 1769074960.000000000, rc -26 00000100:00100000:1.0:1769074962.429661:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913027456:192.168.204.121@tcp:4:cmp.0 00000100:00100000:1.0:1769074962.432207:0:2363:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074962.432234:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913027584:192.168.204.121@tcp:16:kworker.0 00000100:00100000:1.0:1769074962.435954:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913027584:192.168.204.121@tcp:16:kworker.0 00000008:00010000:1.0:1769074962.439436:0:2363:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 00000000931f19d9 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074962.439604:0:2363:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 00000000931f19d9 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935004aa0e00/0x410246667097e241 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97d64 expref: -99 pid: 23704 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074962.439644:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913027456:192.168.204.121@tcp:4:cmp.0 00010000:00010000:1.0:1769074964.662319:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074966.513450:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913028352:192.168.204.121@tcp:63:ll_sa.0 00010000:00010000:1.0:1769074966.520650:0:2363:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520677:0:2363:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520685:0:2363:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520691:0:2363:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520699:0:2363:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520706:0:2363:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935002ea4400 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074966.520711:0:2363:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x3 rrc: 4 type: IBT flags: 0x206400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520719:0:2363:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x3 rrc: 4 type: IBT flags: 0x206400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520726:0:2363:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x3 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520734:0:2363:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074966.520741:0:2363:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x3 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21961 initiator: MDT0 00010000:00010000:1.0:1769074966.520748:0:2363:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4400/0x410246667097e279 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x3/0x3 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21961 initiator: MDT0 00000100:00100000:1.0:1769074966.520785:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913028352:192.168.204.121@tcp:63:ll_sa.0 00000001:02000400:1.0:1769074966.629440:0:24267:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanityn test 14c: open(O_TRUNC) of executing file return -ETXTBSY ============================================================ 04:42:45 (1769074965) 00010000:00010000:1.0:1769074966.706180:0:14430:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074966.706191:0:14430:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.706203:0:14430:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0c00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074966.706209:0:14430:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.706218:0:14430:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.706227:0:14430:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.706309:0:14430:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.706317:0:14430:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074966.706341:0:14430:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.706350:0:14430:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff935004aa0c00/0x410246667097e24f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97d8e expref: -99 pid: 23706 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.706365:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074966.706368:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074966.706370:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074966.706390:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913028736:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074966.710271:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913028736:192.168.204.121@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1769074966.767554:0:24287:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074966.767588:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074966.767609:0:24287:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589934599/1 (0 0) 00010000:00010000:1.0:1769074966.767676:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.767687:0:24287:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.767698:0:24287:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00000100:00100000:1.0:1769074966.767728:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913028864:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074966.767839:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.770032:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913028864:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.770087:0:24287:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770101:0:24287:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770126:0:24287:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770273:0:24287:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770289:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770297:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770304:0:24287:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074966.770316:0:24287:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9350062a7800 x1855008913028864/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074982 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074966.770343:0:24287:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770358:0:24287:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074966.770365:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074966.770394:0:24287:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 12800 00000080:00010000:1.0:1769074966.770401:0:24287:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e280 00000080:00010000:1.0:1769074966.770406:0:24287:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074966.770409:0:24287:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e280 from it ffffa8e541c0bb58 00010000:00010000:1.0:1769074966.770412:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.770421:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.770450:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074966.770472:0:24287:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d14c.sanityn, dir=[0x200000007:0x1:0x0](ffff935011534b08), flags=25 00000080:00200000:1.0:1769074966.770477:0:24287:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d14c.sanityn, dir=[0x200000007:0x1:0x0](ffff935011534b08), intent=0 00000002:00010000:1.0:1769074966.770498:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d14c.sanityn,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074966.770537:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.770545:0:24287:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.770554:0:24287:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00000100:00100000:1.0:1769074966.770575:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913028992:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074966.770655:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.772610:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913028992:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.772636:0:24287:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772663:0:24287:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772670:0:24287:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772688:0:24287:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772700:0:24287:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772710:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772717:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772725:0:24287:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074966.772737:0:24287:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff9350062a7800 x1855008913028992/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074982 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074966.772749:0:24287:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772759:0:24287:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/43 (0 0) 00000080:00200000:1.0:1769074966.772766:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074966.772789:0:24287:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000401:0x2b:0x0], updating i_size 512 00000080:00200000:1.0:1769074966.772793:0:24287:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x2b:0x0](ffff935011632a08): rc = 0 00000080:00010000:1.0:1769074966.772799:0:24287:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b:0x0](ffff935011632a08) for lock 0x410246667097e287 00000080:00010000:1.0:1769074966.772812:0:24287:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b:0x0](ffff935011632a08) 00000080:00010000:1.0:1769074966.772815:0:24287:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e287 from it ffffa8e541c0ba28 00010000:00010000:1.0:1769074966.772819:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.772827:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0000/0x410246667097e287 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e59 expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.772862:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b:0x0](ffff935011632a08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074966.772880:0:24287:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), flags=9 00000080:00200000:1.0:1769074966.772885:0:24287:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), intent=getattr 00000002:00010000:1.0:1769074966.772903:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x0:0x0:0x0]) in obj [0x200000401:0x2b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074966.772944:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.772952:0:24287:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.772960:0:24287:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00000100:00100000:1.0:1769074966.772978:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913029120:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074966.773084:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.775981:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913029120:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.776005:0:24287:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776024:0:24287:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776031:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776041:0:24287:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776096:0:24287:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776103:0:24287:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074966.776108:0:24287:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776115:0:24287:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776123:0:24287:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 5/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776129:0:24287:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074966.776136:0:24287:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776142:0:24287:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e28e lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24287 initiator: MDT0 00000002:00100000:1.0:1769074966.776160:0:24287:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9350062a7800 x1855008913029120/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074982 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074966.776174:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074966.776203:0:24287:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/43 (0 0) 00000080:00200000:1.0:1769074966.776304:0:24287:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074966.776318:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074966.776327:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.776343:0:24287:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.776350:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074966.776355:0:24287:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074966.776358:0:24287:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e280 00000080:00010000:1.0:1769074966.776363:0:24287:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e280 from it ffffa8e541c0bb58 00010000:00010000:1.0:1769074966.776366:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.776387:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.776404:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074966.776419:0:24287:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14c.sanityn, flags=25 00000080:00200000:1.0:1769074966.776425:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b:0x0](ffff935011632a08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074966.776438:0:24287:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), flags=9 00000080:00200000:1.0:1769074966.776442:0:24287:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), intent=getattr 00000002:00010000:1.0:1769074966.776454:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x0:0x0:0x0]) in obj [0x200000401:0x2b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074966.776961:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776970:0:24287:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.776980:0:24287:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00000100:00100000:1.0:1769074966.777004:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913029248:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074966.777221:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.779864:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913029248:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.779884:0:24287:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779895:0:24287:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779903:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779909:0:24287:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779914:0:24287:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779918:0:24287:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0200 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074966.779921:0:24287:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779925:0:24287:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779930:0:24287:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779938:0:24287:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074966.779944:0:24287:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.779948:0:24287:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e295 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24287 initiator: MDT0 00000002:00100000:1.0:1769074966.779963:0:24287:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9350062a7800 x1855008913029248/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074982 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074966.779975:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074966.779998:0:24287:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589935617/43 (0 0) 00000080:00200000:1.0:1769074966.780154:0:24287:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074966.780166:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074966.780174:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.780186:0:24287:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.780192:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074966.780198:0:24287:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074966.780201:0:24287:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e280 00000080:00010000:1.0:1769074966.780206:0:24287:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e280 from it ffffa8e541c0bbf8 00010000:00010000:1.0:1769074966.780209:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.780217:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.780242:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074966.780250:0:24287:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14c.sanityn, flags=25 00000080:00200000:1.0:1769074966.780256:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b:0x0](ffff935011632a08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074966.780269:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b:0x0](ffff935011632a08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074966.780273:0:24287:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), file ffff93500c23d380, open_flags 80c1, mode 81c0 opened 0 00000080:00200000:1.0:1769074966.780282:0:24287:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), intent=open|creat 00000002:00010000:1.0:1769074966.780300:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x200000401:0x2c:0x0]) in obj [0x200000401:0x2b:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1769074966.780362:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.780368:0:24287:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00000020:00100000:1.0:1769074966.780376:0:24287:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074966.780380:0:24287:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7800 x1855008913029504/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074966.780386:0:24287:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00000100:00100000:1.0:1769074966.780402:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913029504:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074966.780519:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.783575:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7800 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913029504:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.783616:0:24287:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 4/0,1 mode: --/CW res: [0x200000401:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.783641:0:24287:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 4/0,1 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.783650:0:24287:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 4/0,1 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.783664:0:24287:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 4/0,1 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.783679:0:24287:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 4/0,1 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.783694:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 5/1,1 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.783700:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 5/1,1 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.783705:0:24287:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074966.783712:0:24287:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff9350062a7800 x1855008913029504/t4294973388(4294973388) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074982 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074966.783721:0:24287:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7800 x1855008913029504/t4294973388(4294973388) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074982 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074966.783733:0:24287:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074966.783742:0:24287:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.783750:0:24287:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/44 (0 0) 00000080:00200000:1.0:1769074966.783755:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0x2b:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074966.783767:0:24287:0:(namei.c:1569:ll_create_it()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), intent=open|creat 00000080:00200000:1.0:1769074966.783798:0:24287:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000401:0x2c:0x0](ffff9350115aa1c8) 00000080:00200000:1.0:1769074966.783867:0:24287:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x2c:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1769074966.783873:0:24287:0:(namei.c:1535:ll_create_node()) setting l_ast_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) 00000080:00010000:1.0:1769074966.783876:0:24287:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) for lock 0x410246667097e29c 00000080:00010000:1.0:1769074966.783886:0:24287:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) 00000080:00200008:1.0:1769074966.783902:0:24287:0:(file.c:1001:ll_file_open()) START file sleep:[0x200000401:0x2c:0x0](ffff9350115aa1c8), flags 100301 00000002:00100000:1.0:1769074966.783922:0:24287:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7800 x1855008913029504/t4294973388(4294973388) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074982 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074966.783957:0:24287:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e29c from it ffff93500693d7e0 00010000:00010000:1.0:1769074966.783961:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.783967:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074966.783984:0:24287:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2c:0x0](ffff9350115aa1c8), inode mode 81c0 mask 40 00000080:00200008:1.0:1769074966.784011:0:24287:0:(file.c:6252:ll_getattr_dentry()) START file sleep:[0x200000401:0x2c:0x0](ffff9350115aa1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074966.784016:0:24287:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c:0x0](ffff9350115aa1c8),name=sleep 00000002:00010000:1.0:1769074966.784031:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c:0x0]) in obj [0x200000401:0x2b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074966.784036:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.784044:0:24287:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074966.784075:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074966.784080:0:24287:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) 00000080:00010000:1.0:1769074966.784082:0:24287:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) for lock 0x410246667097e29c 00000080:00010000:1.0:1769074966.784085:0:24287:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e29c from it ffffa8e541c0bcd8 00010000:00010000:1.0:1769074966.784087:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.784092:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074966.784105:0:24287:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074966.784113:0:24287:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c:0x0] 00000008:00010000:1.0:1769074966.784127:0:24287:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074966.784130:0:24287:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5cc30, osc lock ffff935010593c80, flags 0x400001002 00010000:00010000:1.0:1769074966.784170:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.784176:0:24287:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074966.784207:0:24287:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1769074966.784219:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074966.784244:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913029632:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.786149:0:2362:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.786174:0:2362:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.786190:0:2362:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.786203:0:2362:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074966.786225:0:2362:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074966.786250:0:2362:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1769074966.787332:0:2362:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.787354:0:2362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.787361:0:2362:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.787369:0:2362:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074966.787393:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913029632:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.792367:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074966.792376:0:5554:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.792384:0:5554:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935004aa0600 already unused, calling callback (ffffffffc137c0e0) 00010000:00010000:1.0:1769074966.792390:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074966.792425:0:5554:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074966.792440:0:5554:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074966.792492:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 2/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.792500:0:5554:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074966.792528:0:5554:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 1/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.792536:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2a3 lrc: 0/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf97ead expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.792552:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074966.792554:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074966.792556:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074966.794441:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913029760:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.794475:0:24287:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,1 mode: --/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.794489:0:24287:0:(ldlm_request.c:274:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,1 mode: --/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.794499:0:24287:0:(ldlm_request.c:303:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,1 mode: --/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.794866:0:24287:0:(ldlm_request.c:186:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.794878:0:24287:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.794896:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 5/0,2 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074966.794919:0:24287:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,2 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.794932:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 5/0,2 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.794940:0:24287:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074966.794971:0:24287:0:(vvp_io.c:1312:vvp_io_write_start()) sleep: write [0, 38336) append 0 append lockpos 0 00000080:00200000:1.0:1769074966.795000:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795089:0:24287:0:(rw26.c:912:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795109:0:24287:0:(rw26.c:969:ll_write_end()) page@000000004bc98d35[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795113:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @000000006a9800bf 17ffffc0001001 3:0 ffff9350109daa20 0 lru 00000080:00200000:1.0:1769074966.795120:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@00000000de083724 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795123:0:24287:0:(rw26.c:969:ll_write_end()) end page@000000004bc98d35 00000080:00200000:1.0:1769074966.795124:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1769074966.795128:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795154:0:24287:0:(rw26.c:912:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795158:0:24287:0:(rw26.c:969:ll_write_end()) page@00000000b0cd5e75[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795161:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @00000000f4eef50b 17ffffc0001001 3:0 ffff9350109daca8 1 lru 00000080:00200000:1.0:1769074966.795166:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@00000000102b3cd0 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795169:0:24287:0:(rw26.c:969:ll_write_end()) end page@00000000b0cd5e75 00000080:00200000:1.0:1769074966.795170:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 2. 00000080:00200000:1.0:1769074966.795172:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795203:0:24287:0:(rw26.c:912:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795208:0:24287:0:(rw26.c:969:ll_write_end()) page@0000000022941b1e[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795211:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @0000000030c2dd6b 17ffffc0001001 3:0 ffff9350109daaf8 2 lru 00000080:00200000:1.0:1769074966.795217:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@00000000aca82b4c 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795219:0:24287:0:(rw26.c:969:ll_write_end()) end page@0000000022941b1e 00000080:00200000:1.0:1769074966.795221:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 3. 00000080:00200000:1.0:1769074966.795222:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795251:0:24287:0:(rw26.c:912:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795255:0:24287:0:(rw26.c:969:ll_write_end()) page@0000000046a376a5[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795258:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @00000000b02a6696 17ffffc0001001 3:0 ffff9350109da6c0 3 lru 00000080:00200000:1.0:1769074966.795263:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@000000007ad6415c 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795265:0:24287:0:(rw26.c:969:ll_write_end()) end page@0000000046a376a5 00000080:00200000:1.0:1769074966.795266:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 4. 00000080:00200000:1.0:1769074966.795268:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795288:0:24287:0:(rw26.c:912:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795292:0:24287:0:(rw26.c:969:ll_write_end()) page@000000004b32cec2[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795295:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @00000000e5fb4422 17ffffc0001001 3:0 ffff9350109da438 4 lru 00000080:00200000:1.0:1769074966.795300:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@00000000ac055f21 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795302:0:24287:0:(rw26.c:969:ll_write_end()) end page@000000004b32cec2 00000080:00200000:1.0:1769074966.795303:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 5. 00000080:00200000:1.0:1769074966.795305:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795317:0:24287:0:(rw26.c:912:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795321:0:24287:0:(rw26.c:969:ll_write_end()) page@0000000089579d94[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795324:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @00000000f0056dd4 17ffffc0001001 3:0 ffff9350109da948 5 lru 00000080:00200000:1.0:1769074966.795329:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@0000000014be4103 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795331:0:24287:0:(rw26.c:969:ll_write_end()) end page@0000000089579d94 00000080:00200000:1.0:1769074966.795332:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 6. 00000080:00200000:1.0:1769074966.795334:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795346:0:24287:0:(rw26.c:912:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795350:0:24287:0:(rw26.c:969:ll_write_end()) page@0000000046ca206e[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795353:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @000000002c2c27e2 17ffffc0001001 3:0 ffff9350109dae58 6 lru 00000080:00200000:1.0:1769074966.795358:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@00000000bafc97c4 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795360:0:24287:0:(rw26.c:969:ll_write_end()) end page@0000000046ca206e 00000080:00200000:1.0:1769074966.795361:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 7. 00000080:00200000:1.0:1769074966.795363:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795387:0:24287:0:(rw26.c:912:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795391:0:24287:0:(rw26.c:969:ll_write_end()) page@00000000cbe8a3c9[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795394:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @00000000fde9e08a 17ffffc0001021 2:0 ffff9350109da0d8 7 lru 00000080:00200000:1.0:1769074966.795399:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@000000003c3809e8 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795402:0:24287:0:(rw26.c:969:ll_write_end()) end page@00000000cbe8a3c9 00000080:00200000:1.0:1769074966.795403:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 8. 00000080:00200000:1.0:1769074966.795404:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:1.0:1769074966.795417:0:24287:0:(rw26.c:912:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:1.0:1769074966.795421:0:24287:0:(rw26.c:969:ll_write_end()) page@00000000f61897a0[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795424:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @00000000b5bf238c 17ffffc0001001 3:0 ffff9350109da288 8 lru 00000080:00200000:1.0:1769074966.795429:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@0000000096716726 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795432:0:24287:0:(rw26.c:969:ll_write_end()) end page@00000000f61897a0 00000080:00200000:1.0:1769074966.795433:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 9. 00000080:00200000:1.0:1769074966.795436:0:24287:0:(rw26.c:729:ll_write_begin()) Writing 9 of 0 to 1472 bytes 00000080:00200000:1.0:1769074966.795455:0:24287:0:(rw26.c:912:ll_write_end()) pos 36864, len 1472, copied 1472 00000080:00200000:1.0:1769074966.795458:0:24287:0:(rw26.c:969:ll_write_end()) page@00000000014ee585[3 00000000d92dbdfe 2 1 00000000b546752c] 00000080:00200000:1.0:1769074966.795461:0:24287:0:(rw26.c:969:ll_write_end()) vmpage @0000000008807813 17ffffc0001001 3:0 ffff9350109da5e8 9 lru 00000080:00200000:1.0:1769074966.795466:0:24287:0:(rw26.c:969:ll_write_end()) osc-page@00000000a9f77d3f 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000065e64005 000000007b38d396 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074966.795468:0:24287:0:(rw26.c:969:ll_write_end()) end page@00000000014ee585 00000080:00200000:1.0:1769074966.795470:0:24287:0:(rw26.c:969:ll_write_end()) queued page: 10. 00000080:00200000:1.0:1769074966.795473:0:24287:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 10, from 0, to 1472 prio 0 00000008:00010000:1.0:1769074966.795495:0:24287:0:(osc_cache.c:814:osc_extent_find()) ### extent: 000000009c567a31 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074966.795537:0:24287:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9350115aa378, count 10, dirtied 10 00000080:00200000:1.0:1769074966.795544:0:24287:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 10 pages 38336 bytes, tot: 38336 00000080:00200000:1.0:1769074966.795548:0:24287:0:(vvp_io.c:1390:vvp_io_write_start()) sleep: write bytes 0, result: 38336 00000020:00010000:1.0:1769074966.795566:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074966.795570:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074966.795573:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9350031473b0 (ffff9350053391b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074966.795578:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 5/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.795587:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,0 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074966.795604:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074966.795608:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9350031473b0 (ffff935005339ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074966.795616:0:24287:0:(vvp_io.c:518:vvp_io_advance()) advancing 38336 bytes 00000080:00200000:1.0:1769074966.795623:0:24287:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 9] 00000080:00200000:1.0:1769074966.795632:0:24287:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074966.795636:0:24287:0:(file.c:2142:ll_file_io_generic()) sleep: 2 io complete with rc: 0, result: 38336, restart: 0 00000080:00200000:1.0:1769074966.795644:0:24287:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x2c:0x0] updating i_size 38336 i_blocks 1 00000080:00200000:1.0:1769074966.795655:0:24287:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 38336 00000080:00200008:1.0:1769074966.795697:0:24287:0:(llite_lib.c:2314:ll_setattr_raw()) START file sleep:[0x200000401:0x2c:0x0](ffff9350115aa1c8) current size 38336, valid attrs 101f0, mode ffff, uid 203674496, gid 131072, new size 18446624570743183360, atime 1752421453.000000000, mtime 1674053537.000000000, ctime 1769074966.000000000 00010000:00010000:1.0:1769074966.795737:0:24287:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074966.795756:0:24287:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x2c:0x0]: losing layout lock 00010000:00010000:1.0:1769074966.795800:0:24287:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.795807:0:24287:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074966.795810:0:24287:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e29c lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x48bf02eaaaf97e9f expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074966.795831:0:24287:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074966.795840:0:24287:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7480 x1855008913030144/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074966.795858:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030144:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074966.795913:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.798523:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7480 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030144:192.168.204.121@tcp:36:cp.0 00000080:00200000:1.0:1769074966.798574:0:24287:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:1.0:1769074966.798598:0:24287:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0x2c:0x0](ffff9350115aa1c8) 00000002:00010000:1.0:1769074966.798611:0:24287:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c:0x0]) in obj [0x200000401:0x2c:0x0], intent: layout flags 0 00010000:00010000:1.0:1769074966.798671:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.798681:0:24287:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00010000:00010000:1.0:1769074966.798693:0:24287:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 24287 initiator: MDT0 00000100:00100000:1.0:1769074966.798713:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030272:192.168.204.121@tcp:101:cp.0 00000100:00100000:1.0:1769074966.798897:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.801411:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6a00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030272:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074966.801447:0:24287:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.801465:0:24287:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1769074966.801485:0:24287:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9350062a6a00 x1855008913030272/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 376/400 e 0 to 0 dl 1769074982 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074966.801497:0:24287:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000002:00010000:1.0:1769074966.801515:0:24287:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.801529:0:24287:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 16 res 8589935617/44 (0 0) 00000080:00200000:1.0:1769074966.801536:0:24287:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0x2c:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1769074966.801555:0:24287:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) for lock 0x410246667097e2b1 00000080:00010000:1.0:1769074966.801564:0:24287:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e2b1 from it ffffa8e541c0ba78 00010000:00010000:1.0:1769074966.801568:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.801578:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.801599:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.801611:0:24287:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1769074966.801620:0:24287:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000401:0x2c:0x0](00000000e2c961cc) being reconfigured ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074966.801662:0:24287:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x2c:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:1.0:1769074966.801669:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.801676:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0200/0x410246667097e2b1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97ec2 expref: -99 pid: 24287 timeout: 0 lvb_type: 3 00020000:00200000:1.0:1769074966.801696:0:24287:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074966.801711:0:24287:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1769074966.801728:0:24287:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074966.801732:0:24287:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c9b0, osc lock ffff935010593db0, flags 0x0 00010000:00010000:1.0:1769074966.801753:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.801762:0:24287:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.801774:0:24287:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 7/0,2 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.801783:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 7/0,2 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.801789:0:24287:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 6/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074966.806919:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074966.806926:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074966.806934:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5c9b0 (ffff9350053391b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074966.806948:0:24287:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 5/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.806965:0:24287:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 4/0,0 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074966.806985:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a248a8 (ffff935005339ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074966.806989:0:24287:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5c9b0 (ffff935005339ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074966.807010:0:24287:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:1.0:1769074966.807029:0:24287:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff93500936c600 uarg=0000000000000000 00000002:00000080:1.0:1769074966.807041:0:24287:0:(mdc_request.c:2279:mdc_iocontrol()) lustre-MDT0000-mdc-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff93500936c600 uarg=0000000000000000 00000020:00100000:1.0:1769074966.807095:0:24287:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:1.0:1769074966.807104:0:24287:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4000 x1855008913030656/t0(0) o55->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074966.807127:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030656:192.168.204.121@tcp:55:cp.0 00000100:00100000:1.0:1769074966.807240:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.811358:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030656:192.168.204.121@tcp:55:cp.0 00000080:00200008:1.0:1769074966.811462:0:24287:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file sleep:[0x200000401:0x2c:0x0](ffff9350115aa1c8) current size 38336, valid attrs 101f0, mode ffff, uid 203674496, gid 131072, new size 18446624570743183360, atime 1752421453.000000000, mtime 1674053537.000000000, ctime 1769074966.000000000, rc 0 00000020:00100000:1.0:1769074966.811602:0:24287:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074966.811612:0:24287:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4000 x1855008913030912/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 544/440 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074966.811635:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030912:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074966.811727:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.814351:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4000 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913030912:192.168.204.121@tcp:36:cp.0 00000080:00200008:1.0:1769074966.814418:0:24287:0:(file.c:424:ll_file_release()) START file sleep:[0x200000401:0x2c:0x0](ffff9350115aa1c8), flags 100001 00000002:00100000:1.0:1769074966.814473:0:24287:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350062a7800 x1855008913029504/t4294973388(4294973388) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074982 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074966.814485:0:24287:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074966.814489:0:24287:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4e00 x1855008913031168/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074966.814505:0:24287:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913031168:192.168.204.121@tcp:35:cp.0 00000100:00100000:1.0:1769074966.814843:0:24287:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.818387:0:24287:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4e00 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24287:1855008913031168:192.168.204.121@tcp:35:cp.0 00000002:00100000:1.0:1769074966.818428:0:24287:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9350062a7800 x1855008913029504/t4294973388(4294973388) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074982 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074966.821963:0:24288:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769074966.821999:0:24288:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074966.822019:0:24288:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.822036:0:24288:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.822043:0:24288:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074966.822078:0:24288:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) 00000080:00010000:1.0:1769074966.822082:0:24288:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff935011534b08) for lock 0x410246667097e280 00000080:00010000:1.0:1769074966.822090:0:24288:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e280 from it ffffa8e541c63b88 00010000:00010000:1.0:1769074966.822095:0:24288:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.822120:0:24288:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0c00/0x410246667097e280 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97e3d expref: -99 pid: 24287 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074966.822138:0:24288:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08), inode mode 41ff mask 1 00000080:00200000:1.0:1769074966.822147:0:24288:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14c.sanityn, flags=25 00000080:00200000:1.0:1769074966.822153:0:24288:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b:0x0](ffff935011632a08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074966.822166:0:24288:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), file ffff93500c23c600, open_flags 8020, mode 0 opened 0 00000080:00200000:1.0:1769074966.822172:0:24288:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=sleep, dir=[0x200000401:0x2b:0x0](ffff935011632a08), intent=open 00000002:00010000:1.0:1769074966.822188:0:24288:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sleep,[0x0:0x0:0x0]) in obj [0x200000401:0x2b:0x0], intent: open flags 0100041 00010000:00010000:1.0:1769074966.822264:0:24288:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24288 initiator: MDT0 00010000:00010000:1.0:1769074966.822272:0:24288:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24288 initiator: MDT0 00000020:00100000:1.0:1769074966.822285:0:24288:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074966.822299:0:24288:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7b80 x1855008913031424/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074966.822309:0:24288:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24288 initiator: MDT0 00000100:00100000:1.0:1769074966.822335:0:24288:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job bash:7afd476b-94ee-4c94-b080-d15824f3ae56:24288:1855008913031424:192.168.204.121@tcp:101:bash.0 00000100:00100000:1.0:1769074966.822587:0:24288:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.826305:0:24288:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job bash:7afd476b-94ee-4c94-b080-d15824f3ae56:24288:1855008913031424:192.168.204.121@tcp:101:bash.0 00010000:00010000:1.0:1769074966.826338:0:24288:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.826363:0:24288:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.826371:0:24288:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.826384:0:24288:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.826395:0:24288:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.826407:0:24288:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.826415:0:24288:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.826423:0:24288:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074966.826434:0:24288:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9350062a7b80 x1855008913031424/t4294973392(4294973392) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074982 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074966.826448:0:24288:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7b80 x1855008913031424/t4294973392(4294973392) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074982 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074966.826462:0:24288:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074966.826482:0:24288:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.826494:0:24288:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/44 (0 0) 00000080:00200000:1.0:1769074966.826501:0:24288:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x2b:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074966.826551:0:24288:0:(namei.c:151:ll_iget()) got inode: [0x200000401:0x2c:0x0](ffff9350115aa1c8): rc = 0 00000080:00200000:1.0:1769074966.826567:0:24288:0:(file.c:6809:ll_layout_conf()) [0x200000401:0x2c:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1769074966.826572:0:24288:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) for lock 0x410246667097e2b8 00000080:00010000:1.0:1769074966.826585:0:24288:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) 00000080:00200008:1.0:1769074966.826601:0:24288:0:(file.c:1001:ll_file_open()) START file sleep:[0x200000401:0x2c:0x0](ffff9350115aa1c8), flags 100040 00000002:00100000:1.0:1769074966.826624:0:24288:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9350062a7b80 x1855008913031424/t4294973392(4294973392) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074982 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074966.826665:0:24288:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e2b8 from it ffff93500693d540 00010000:00010000:1.0:1769074966.826669:0:24288:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074966.826679:0:24288:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1200/0x410246667097e2b8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97eec expref: -99 pid: 24288 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074966.826703:0:24288:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2c:0x0](ffff9350115aa1c8), inode mode 81ed mask 41 00000002:00010000:1.0:1769074966.826956:0:24288:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000401:0x2c:0x0], intent: getxattr flags 0 00010000:00010000:1.0:1769074966.827007:0:24288:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24288 initiator: MDT0 00010000:00010000:1.0:1769074966.827013:0:24288:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24288 initiator: MDT0 00010000:00010000:1.0:1769074966.827020:0:24288:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24288 initiator: MDT0 00000100:00100000:1.0:1769074966.827040:0:24288:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job bash:7afd476b-94ee-4c94-b080-d15824f3ae56:24288:1855008913031552:192.168.204.121@tcp:101:bash.0 00000100:00100000:1.0:1769074966.827159:0:24288:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074966.831482:0:24288:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job bash:7afd476b-94ee-4c94-b080-d15824f3ae56:24288:1855008913031552:192.168.204.121@tcp:101:bash.0 00010000:00010000:1.0:1769074966.831500:0:24288:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97f08 expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.831512:0:24288:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97f08 expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.831521:0:24288:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97f08 expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074966.831541:0:24288:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff9350062a4a80 x1855008913031552/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 568/1168 e 0 to 0 dl 1769074982 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074966.831559:0:24288:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97f08 expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.831584:0:24288:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 4 res 8589935617/44 (0 0) 00000080:00200000:1.0:1769074966.831590:0:24288:0:(namei.c:955:ll_intent_lock()) intent lock 128 on i1 [0x200000401:0x2c:0x0] suppgids 0 -1: rc 0 00000080:00010000:1.0:1769074966.831634:0:24288:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c:0x0](ffff9350115aa1c8) for lock 0x410246667097e2bf 00000080:00010000:1.0:1769074966.831642:0:24288:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x410246667097e2bf from it ffffa8e541c63ba0 00010000:00010000:1.0:1769074966.831646:0:24288:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf97f08 expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074966.831653:0:24288:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0600/0x410246667097e2bf lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf97f08 expref: -99 pid: 24288 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074966.831690:0:24288:0:(file.c:2474:do_file_read_iter()) START file sleep:[0x200000401:0x2c:0x0], ppos: 0, count: 256 00000080:00200000:1.0:1769074966.831705:0:24288:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074966.831733:0:24288:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074966.831777:0:24288:0:(file.c:2474:do_file_read_iter()) START file sleep:[0x200000401:0x2c:0x0], ppos: 64, count: 616 00000080:00200000:1.0:1769074966.831782:0:24288:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074966.831788:0:24288:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074966.831800:0:24288:0:(file.c:2474:do_file_read_iter()) START file sleep:[0x200000401:0x2c:0x0], ppos: 680, count: 28 00000080:00200000:1.0:1769074966.831804:0:24288:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074966.831808:0:24288:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074966.831877:0:24288:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2c:0x0](ffff9350115aa1c8), inode mode 81ed mask 4 00000080:00a00000:1.0:1769074966.832748:0:24288:0:(llite_mmap.c:598:ll_file_mmap()) VFS_Op: fid=[0x200000401:0x2c:0x0] vma=ffff9350095b49f8 start=0x55a04d5dd000 end=0x55a04d7e6000 vm_flags=0x875 00000080:00200000:1.0:1769074966.832761:0:24288:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074966.832768:0:24288:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c:0x0] 00000008:00010000:1.0:1769074966.832789:0:24288:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074966.832793:0:24288:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c330, osc lock ffff935010593db0, flags 0x400001002 00010000:00010000:1.0:1769074966.832814:0:24288:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.832829:0:24288:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074966.832841:0:24288:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074966.832846:0:24288:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 6/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.832854:0:24288:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 5/0,0 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074966.832867:0:24288:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074966.832877:0:24288:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x2c:0x0] updating i_size 38336 i_blocks 1 00000020:00010000:1.0:1769074966.832881:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074966.832884:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074966.832886:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5c330 (ffff9350053391b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074966.832891:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074966.832893:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5c330 (ffff935005339ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074966.832901:0:24288:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00a00000:1.0:1769074966.832930:0:24288:0:(llite_mmap.c:598:ll_file_mmap()) VFS_Op: fid=[0x200000401:0x2c:0x0] vma=ffff9350095b4740 start=0x55a04d7e4000 end=0x55a04d7e6000 vm_flags=0x100873 00000080:00200000:1.0:1769074966.832937:0:24288:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074966.832941:0:24288:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c:0x0] 00000008:00010000:1.0:1769074966.832947:0:24288:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074966.832949:0:24288:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff935010d5c330, osc lock ffff935010593db0, flags 0x400001002 00010000:00010000:1.0:1769074966.832955:0:24288:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.832963:0:24288:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074966.832970:0:24288:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074966.832972:0:24288:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 6/0,1 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.832979:0:24288:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 5/0,0 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074966.832992:0:24288:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074966.832996:0:24288:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000401:0x2c:0x0] updating i_size 38336 i_blocks 1 00000020:00010000:1.0:1769074966.832998:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074966.833000:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074966.833003:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935010d5c330 (ffff9350053391b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074966.833005:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93500848a480 (ffff935005339ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074966.833007:0:24288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935010d5c330 (ffff935005339ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074966.833012:0:24288:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2c:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1769074966.849183:0:2363:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 6/2,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849211:0:2363:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 6/2,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849226:0:2363:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 6/2,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849244:0:2363:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 6/2,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849253:0:2363:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 5/1,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074966.849269:0:2363:0:(osc_lock.c:179:osc_lock_lvb_update()) ### no dlm lock: acquired size 38336, setting rss=38336; leaving kms=0, end=18446744073709551615 00000020:00200000:1.0:1769074966.849276:0:2363:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1769074966.849289:0:2363:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 4/1,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849294:0:2363:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 3/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849299:0:2363:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 4/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849304:0:2363:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935002ea4200 already unused, calling callback (ffffffffc137c0e0) 00010000:00010000:1.0:1769074966.849309:0:2363:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 5/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849319:0:2363:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074966.849324:0:2363:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 3/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074966.849328:0:2363:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff935002ea4200/0x410246667097e2db lrc: 0/0,0 mode: --/PR res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 24289 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074966.849353:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913032064:192.168.204.121@tcp:101:cp.0 00000080:00200008:1.0:1769074966.862563:0:24288:0:(file.c:424:ll_file_release()) START file sleep:[0x200000401:0x2c:0x0](ffff9350115aa1c8), flags 100040 00000002:00100000:1.0:1769074966.862641:0:24288:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350062a7b80 x1855008913031424/t4294973392(4294973392) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074982 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074966.862653:0:24288:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074966.862659:0:24288:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a4a80 x1855008913032704/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074966.862690:0:24288:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job sleep:7afd476b-94ee-4c94-b080-d15824f3ae56:24288:1855008913032704:192.168.204.121@tcp:35:sleep.0 00000100:00100000:1.0:1769074966.862787:0:24288:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0d80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074968.049413:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:987cd90b-10bf-40f3-bb36-40b816fac7c1:2363:1855008913033088:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.049546:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913033344:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.049575:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913033600:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.049600:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913033856:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.049629:0:2363:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1769074968.049635:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913033216:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.049657:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913033472:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.049679:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913033728:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.052363:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:987cd90b-10bf-40f3-bb36-40b816fac7c1:2363:1855008913033088:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.052391:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9350067ee000: committing for last_committed 4294967343 gen 4 00000100:00100000:1.0:1769074968.052429:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913033344:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.052444:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913033600:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.052457:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913033856:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.052479:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913033216:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.052492:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913033472:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074968.052503:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913033728:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769074968.755269:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074969.614430:0:5555:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074969.614441:0:5555:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316ba00/0x410246667097e2c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x48bf02eaaaf97f0f expref: -99 pid: 24289 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074969.614454:0:5555:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff93500316ba00 already unused, calling callback (ffffffffc1cc3b30) 00010000:00010000:1.0:1769074969.614459:0:5555:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316ba00/0x410246667097e2c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97f0f expref: -99 pid: 24289 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074969.614469:0:5555:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316ba00/0x410246667097e2c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x48bf02eaaaf97f0f expref: -99 pid: 24289 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074969.614477:0:5555:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316ba00/0x410246667097e2c6 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x48bf02eaaaf97f0f expref: -99 pid: 24289 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074969.614544:0:5555:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316ba00/0x410246667097e2c6 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97f0f expref: -99 pid: 24289 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074969.614551:0:5555:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074969.614573:0:5555:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316ba00/0x410246667097e2c6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97f0f expref: -99 pid: 24289 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074969.614580:0:5555:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff935006f53800 lock: ffff93500316ba00/0x410246667097e2c6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x48bf02eaaaf97f0f expref: -99 pid: 24289 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074969.614591:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074969.614594:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074969.614596:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074969.614612:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913034240:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074969.615719:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913034240:192.168.204.121@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074969.663559:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913035136:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074969.664611:0:2363:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 6/2,0 mode: --/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.664628:0:2363:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 6/2,0 mode: --/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.664642:0:2363:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 6/2,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.664652:0:2363:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 6/2,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074969.664666:0:2363:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 5/2,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074969.664681:0:2363:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1769074969.664693:0:2363:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 6/2,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.664701:0:2363:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 5/1,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.664709:0:2363:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 5/1,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.664719:0:2363:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 4/0,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074969.664735:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913035136:192.168.204.121@tcp:101:cp.0 00010000:00010000:1.0:1769074969.666323:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074969.666332:0:5554:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 2/0,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.666342:0:5554:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff935002ea4e00 already unused, calling callback (ffffffffc137c0e0) 00010000:00010000:1.0:1769074969.666347:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 3/0,0 mode: PR/PR res: [0x240000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074969.666363:0:5554:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x2e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074969.666373:0:5554:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074969.666406:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 2/0,0 mode: --/PR res: [0x240000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.666413:0:5554:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074969.666443:0:5554:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 1/0,0 mode: --/PR res: [0x240000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.666451:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea4e00/0x410246667097e31a lrc: 0/0,0 mode: --/PR res: [0x240000400:0xc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x48bf02eaaaf98004 expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.666466:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074969.666469:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074969.666473:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00010000:1.0:1769074969.672374:0:24869:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074969.672380:0:24869:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074969.672382:0:24869:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9350123778b0 (ffff935005339bd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074969.672389:0:24869:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e321 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x48bf02eaaaf9800b expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074969.672410:0:24869:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9350067ee000 lock: ffff935002ea5800/0x410246667097e321 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x48bf02eaaaf9800b expref: -99 pid: 24869 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074969.672424:0:24869:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff935002a243f8 (ffff9350053393f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074969.672426:0:24869:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9350123778b0 (ffff9350053393f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074969.672442:0:24869:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x2e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:1.0:1769074969.672455:0:24869:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff9350084c3000 uarg=0000000000000000 00000002:00000080:1.0:1769074969.672465:0:24869:0:(mdc_request.c:2279:mdc_iocontrol()) lustre-MDT0000-mdc-ffff9350067ee000: cmd=401866d4 len=384 karg=ffff9350084c3000 uarg=0000000000000000 00000020:00100000:1.0:1769074969.672487:0:24869:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:1.0:1769074969.672495:0:24869:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008913036160/t0(0) o55->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074969.672514:0:24869:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24869:1855008913036160:192.168.204.121@tcp:55:cp.0 00000100:00100000:1.0:1769074969.672557:0:24869:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0a80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074969.673569:0:24869:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24869:1855008913036160:192.168.204.121@tcp:55:cp.0 00000080:00200008:1.0:1769074969.673597:0:24869:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file sleep:[0x200000401:0x2e:0x0](ffff9350116300c8) current size 38336, valid attrs 101f0, mode ffff, uid 103551488, gid 131072, new size 18446624570643058048, atime 1752421453.000000000, mtime 1674053537.000000000, ctime 1769074969.000000000, rc 0 00000020:00100000:1.0:1769074969.673687:0:24869:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074969.673694:0:24869:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7100 x1855008913036416/t0(0) o36->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 544/440 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074969.673714:0:24869:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24869:1855008913036416:192.168.204.121@tcp:36:cp.0 00000100:00100000:1.0:1769074969.673762:0:24869:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0a80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074969.675192:0:24869:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7100 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24869:1855008913036416:192.168.204.121@tcp:36:cp.0 00000080:00200008:1.0:1769074969.675240:0:24869:0:(file.c:424:ll_file_release()) START file sleep:[0x200000401:0x2e:0x0](ffff9350116300c8), flags 100001 00000002:00100000:1.0:1769074969.675281:0:24869:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350103f9880 x1855008913035008/t4294973396(4294973396) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074985 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074969.675289:0:24869:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074969.675292:0:24869:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a7b80 x1855008913036672/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074969.675305:0:24869:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24869:1855008913036672:192.168.204.121@tcp:35:cp.0 00000100:00100000:1.0:1769074969.675353:0:24869:0:(client.c:2690:ptlrpc_set_wait()) set ffff935004ab0a80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074969.676455:0:24869:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a7b80 pname:cluuid:pid:xid:nid:opc:job cp:7afd476b-94ee-4c94-b080-d15824f3ae56:24869:1855008913036672:192.168.204.121@tcp:35:cp.0 00000002:00100000:1.0:1769074969.676480:0:24869:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9350103f9880 x1855008913035008/t4294973396(4294973396) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074985 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1769074969.777166:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074970.165730:0:24996:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: chmod 00010000:00010000:1.0:1769074970.802235:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074971.825279:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074972.849240:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074973.169344:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:987cd90b-10bf-40f3-bb36-40b816fac7c1:2362:1855008913037440:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074973.169467:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913037696:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074973.169509:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913037952:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074973.169559:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913038208:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769074973.873294:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074974.897216:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074975.921279:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074976.945293:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074977.969286:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074978.289279:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:987cd90b-10bf-40f3-bb36-40b816fac7c1:2363:1855008913038336:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074978.289409:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913038592:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074978.289462:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913038848:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769074978.993263:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074980.017207:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074981.041295:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074982.065280:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074983.089223:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074984.113167:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074985.137222:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074986.161157:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074987.185194:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074988.209169:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074988.529209:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:987cd90b-10bf-40f3-bb36-40b816fac7c1:2362:1855008913040128:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074988.529317:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913040384:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074988.529352:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074988.529356:0:2362:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913040256:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769074989.233191:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074989.553163:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074973/real 1769074973] req@ffff9350103fb800 x1855008913037440/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.571966:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103fb800 x1855008913037440/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074989.571978:0:2362:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1769074989.571981:0:2362:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.121@tcp: Connection to MGS (at 192.168.204.121@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1769074989.576558:0:2362:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.121@tcp for MGC192.168.204.121@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1769074989.576562:0:2362:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1769074989.576567:0:2362:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff9350103fb800 x1855008913037440/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.576577:0:2362:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff9350103fbb80 x1855008913038336/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074994 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.576586:0:2362:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff935002a61c00 x1855008913039232/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074999 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.576599:0:2362:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff9350050bd180 x1855008913040128/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1769075004 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.576608:0:2362:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff935002a7d180 x1855008913041024/t0(0) o400->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 224/224 e 0 to 0 dl 1769075005 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1769074989.576624:0:2362:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1769074989.576627:0:2362:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1769074989.576630:0:2362:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1769074989.576632:0:2362:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:1.0:1769074989.576665:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074973/real 1769074973] req@ffff9350103f8380 x1855008913037696/t0(0) o400->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.576692:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103f8380 x1855008913037696/t0(0) o400->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074989.576715:0:2362:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074989.576719:0:2362:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff9350067ee000: Connection to lustre-OST0000 (at 192.168.204.121@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074989.576724:0:2362:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00000400:1.0:1769074989.576744:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074973/real 1769074973] req@ffff9350103f8000 x1855008913037952/t0(0) o400->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.576792:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103f8000 x1855008913037952/t0(0) o400->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074989.576820:0:2362:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074989.576836:0:2362:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff935006f53800: Connection to lustre-MDT0000 (at 192.168.204.121@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074989.576843:0:2362:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:1.0:1769074989.576887:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074973/real 1769074973] req@ffff9350103f9180 x1855008913038208/t0(0) o400->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074989.576947:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103f9180 x1855008913038208/t0(0) o400->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074989 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074989.576983:0:2362:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074989.576995:0:2362:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff935006f53800: Connection to lustre-OST0001 (at 192.168.204.121@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074989.577003:0:2362:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00100000:1.0:1769074989.577032:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:987cd90b-10bf-40f3-bb36-40b816fac7c1:2362:1855008913040128:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074989.577046:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:987cd90b-10bf-40f3-bb36-40b816fac7c1:2362:1855008913037440:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074989.577115:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913037952:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769074990.257216:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074991.281236:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074992.305197:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074993.329201:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074994.353255:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074994.737210:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074978/real 1769074978] req@ffff9350103fb100 x1855008913039104/t0(0) o400->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074994.737254:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103fb100 x1855008913039104/t0(0) o400->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074994.737278:0:2362:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9350052d9000 already not connected (conn 4, was 3): CONNECTING 00000100:00000400:1.0:1769074994.737287:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074978/real 1769074978] req@ffff9350103fad80 x1855008913038464/t0(0) o400->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074994.737304:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103fad80 x1855008913038464/t0(0) o400->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074994.737312:0:2362:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9350052ab800 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:1.0:1769074994.737317:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074978/real 1769074978] req@ffff9350103fa680 x1855008913038720/t0(0) o400->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074994.737330:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103fa680 x1855008913038720/t0(0) o400->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074994.737335:0:2362:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9350052aa800 already not connected (conn 3, was 2): CONNECTING 00000100:00000400:1.0:1769074994.737339:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074978/real 1769074978] req@ffff9350103f9f80 x1855008913038976/t0(0) o400->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074994.737346:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350103f9f80 x1855008913038976/t0(0) o400->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074994 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074994.737352:0:2362:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9350052d8000 already not connected (conn 3, was 2): CONNECTING 00000100:00100000:1.0:1769074994.737368:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913038464:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769074995.377264:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074996.401259:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074997.425280:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074998.449233:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074998.705284:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913037824:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769074999.473230:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074999.729419:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913039104:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074999.729441:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913038208:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074999.729558:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7afd476b-94ee-4c94-b080-d15824f3ae56:2363:1855008913038592:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074999.729582:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913037696:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074999.729667:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:84af2899-c956-435d-9e3c-5d5fbc3c9246:2363:1855008913038080:192.168.204.121@tcp:400:kworker.0 00000100:00100000:1.0:1769074999.729717:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:84af2899-c956-435d-9e3c-5d5fbc3c9246:2362:1855008913038976:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769075000.497265:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075001.521224:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075002.545307:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075003.569226:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075004.593301:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769075004.849177:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074988/real 1769074988] req@ffff9350050bc000 x1855008913040256/t0(0) o400->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075004 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075004.849252:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350050bc000 x1855008913040256/t0(0) o400->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075004 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075004.849261:0:2362:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9350052ab800 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1769075004.849270:0:2362:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074988/real 1769074988] req@ffff9350050bf100 x1855008913040384/t0(0) o400->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075004 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075004.849283:0:2362:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9350050bf100 x1855008913040384/t0(0) o400->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075004 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075004.849291:0:2362:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9350052ac000 already not connected (conn 6, was 4): CONNECTING 00000100:00100000:1.0:1769075004.849308:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913040256:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769075005.617199:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075006.641276:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075007.665231:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075008.689284:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075010.737337:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075011.761313:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075012.785222:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075014.001383:0:2362:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7afd476b-94ee-4c94-b080-d15824f3ae56:2362:1855008913040384:192.168.204.121@tcp:400:kworker.0 00010000:00010000:1.0:1769075014.833204:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075015.857160:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075016.881214:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075018.929293:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075019.249221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075019.249233:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075019.249237:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075019.249242:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075019.249246:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075019.249249:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075019.249251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075019.249254:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075019.249257:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075019.249259:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075019.249262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075019.249265:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075019.249267:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075019.249270:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1769075019.953305:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075020.977284:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075023.025233:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075024.049339:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075024.369171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075024.369184:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075024.369188:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075024.369193:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075024.369196:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075024.369199:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075024.369203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075024.369206:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075024.369209:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075024.369213:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075024.369216:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075024.369224:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075024.369227:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075024.369230:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1769075025.073206:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075027.121236:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075028.145350:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075029.169228:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075031.217264:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075032.241253:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075033.265313:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075035.313261:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075035.633373:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.121:988 00000400:00000100:1.0:1769075035.633421:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075035.633441:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075035.633488:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075035.633499:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075035.633508:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075035.633520:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075035.633530:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075035.633540:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00010000:00010000:1.0:1769075036.337247:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075037.361297:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075039.409275:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075040.433311:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075041.457305:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075041.777372:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.121:988 00000400:00000100:1.0:1769075041.777418:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075041.777436:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075041.777471:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075041.777477:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075041.777487:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075041.777498:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075041.777506:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075041.777519:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00010000:00010000:1.0:1769075043.505241:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075044.529191:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075045.553275:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075047.601308:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075047.921350:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.121:988 00000400:00000100:1.0:1769075047.921397:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075047.921415:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075047.921449:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075047.921456:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075047.921463:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075047.921469:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075047.921474:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075047.921479:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00010000:00010000:1.0:1769075048.625222:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075049.649265:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075051.697298:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075052.721308:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075053.745280:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075055.793237:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075056.817245:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075057.137460:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.121:988 00000400:00000100:1.0:1769075057.137520:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075057.137559:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075057.137607:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075057.137624:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075057.137633:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075057.137643:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075057.137651:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075057.137659:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00010000:00010000:1.0:1769075057.841270:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075059.889197:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075060.913274:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075061.937282:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075063.985192:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075065.009290:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075065.329567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075065.329569:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075065.329574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 771 00000100:00080000:3.0:1769075065.329599:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075065.329604:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075065.329610:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075065.329624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d880 x1855008913050496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1769075066.033207:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075066.353411:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.121:988 00000400:00000100:1.0:1769075066.353460:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075066.353481:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075066.353521:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075066.353535:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075066.353548:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075066.353576:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075066.353592:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075066.353614:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00010000:00010000:3.0:1769075067.057267:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075068.081257:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075069.105286:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075070.129188:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075071.153187:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075072.177206:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075073.201205:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075074.225286:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075075.249233:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075075.569226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075075.569236:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075075.569241:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075075.569252:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 781 00000100:00080000:3.0:1769075075.569264:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075075.569273:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075075.569312:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913051520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075075.569351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075075.569354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075075.569356:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075075.569361:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 781 00000100:00080000:3.0:1769075075.569365:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075075.569369:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075075.569380:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075075.569399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913051648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075075.569435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075075.569458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075075.569461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075075.569472:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 781 00000100:00080000:3.0:1769075075.569479:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075075.569486:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075075.569518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075075.569539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913051776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075075.569548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075075.569552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075075.569554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075075.569558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 781 00000100:00080000:3.0:1769075075.569573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075075.569577:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075075.569581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075075.569591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913051904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075075.569597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075075.569605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075075.569608:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075075.569616:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 781 00000100:00080000:3.0:1769075075.569626:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075075.569640:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075075.569650:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075075.569657:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913052032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075075.569664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075075.569669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075075.569671:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075075.569682:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 781 00000100:00080000:3.0:1769075075.569686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075075.569697:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075075.569705:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075075.569718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913052160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075075.569725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075075.569734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075075.569737:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075075.569747:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 781 00000100:00080000:3.0:1769075075.569756:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075075.569765:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075075.569777:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075075.569795:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913052288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1769075076.273254:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075077.297280:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075078.321192:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075079.345215:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075080.369227:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075081.393260:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075082.417243:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075083.441219:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075084.465241:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075085.489168:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075086.513293:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075087.537307:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075087.857484:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075087.857532:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075087.857550:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075087.857589:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075087.857603:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075087.857617:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075087.857628:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075087.857645:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075087.857661:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00010000:00010000:1.0:1769075088.561290:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075089.585300:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075090.609289:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075090.929421:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913054208:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075090.929497:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075090.929537:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff935002a60a80 x1855008913054208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075090.929559:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913054208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075090.929579:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075090.929588:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075090.929597:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913054208:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075090.929633:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913054336:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075090.929670:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075090.929697:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff935002a61180 x1855008913054336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075090.929743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913054336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075090.929757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075090.929765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075090.929772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913054336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075090.929784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913054464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075090.929824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075090.929844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff935002a63100 x1855008913054464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075090.929859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913054464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075090.929867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075090.929870:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075090.929874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913054464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075090.929881:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913054592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075090.929900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075090.929911:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff935002a61f80 x1855008913054592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075090.929922:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913054592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075090.929930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075090.929933:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075090.929936:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913054592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075090.929953:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913054720:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075090.929973:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075090.929985:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff935002a60a80 x1855008913054720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075090.929999:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913054720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075090.930006:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075090.930010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075090.930013:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913054720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075090.930019:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913054848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075090.930039:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075090.930079:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff935002a62300 x1855008913054848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075090.930099:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913054848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075090.930107:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075090.930110:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075090.930113:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913054848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075090.930121:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913054976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075090.930143:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075090.930173:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff935002a61500 x1855008913054976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075090.930265:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913054976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075090.930281:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075090.930284:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075090.930289:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913054976:192.168.204.121@tcp:8:kworker.0 00010000:00010000:3.0:1769075091.633286:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075092.657287:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075093.681238:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075094.705218:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075095.729282:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075096.049435:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913055104:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075096.049542:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075096.049604:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075096/real 1769075096] req@ffff9350050bed80 x1855008913055104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075096.049642:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bed80 x1855008913055104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075096.049665:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075096.049677:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075096.049686:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913055104:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075096.049732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913055232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075096.049774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075096.049801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075096/real 1769075096] req@ffff9350050bd500 x1855008913055232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075096.049820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913055232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075096.049831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075096.049838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075096.049842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913055232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075096.049850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913055360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075096.049885:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075096.049898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075096/real 1769075096] req@ffff9350050bca80 x1855008913055360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075096.049913:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913055360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075096.049923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075096.050318:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075096.050325:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913055360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075096.050335:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913055488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075096.050368:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075096.050385:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075096/real 1769075096] req@ffff9350050bfb80 x1855008913055488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075096.050399:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913055488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075096.050409:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075096.050414:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075096.050418:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913055488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075096.050425:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913055616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075096.050450:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075096.050464:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075096/real 1769075096] req@ffff9350050bf480 x1855008913055616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075096.050488:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913055616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075096.050498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075096.050502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075096.050507:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913055616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075096.050514:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913055744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075096.050539:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075096.050551:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075096/real 1769075096] req@ffff9350050bf100 x1855008913055744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075096.050566:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913055744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075096.050575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075096.050579:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075096.050581:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913055744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075096.050604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913055872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075096.050629:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075096.050641:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075096/real 1769075096] req@ffff9350050bc000 x1855008913055872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075096.050664:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913055872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075096.050672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075096.050676:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075096.050681:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913055872:192.168.204.121@tcp:8:kworker.0 00010000:00010000:1.0:1769075096.753243:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075097.777285:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075098.801208:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075099.825230:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075100.849224:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075101.169537:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913056000:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075101.169629:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075101.169702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075101/real 1769075101] req@ffff9350050bd180 x1855008913056000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075101.169744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd180 x1855008913056000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075101.169772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075101.169793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075101.169809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913056000:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075101.169859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913056128:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075101.169904:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075101.169931:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075101/real 1769075101] req@ffff9350050bc000 x1855008913056128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075101.170005:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913056128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075101.170183:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075101.170189:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075101.170198:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913056128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075101.170207:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913056256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075101.170238:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075101.170251:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075101/real 1769075101] req@ffff9350050bf100 x1855008913056256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075101.170263:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913056256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075101.170271:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075101.170275:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075101.170278:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913056256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075101.170285:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913056384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075101.170304:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075101.170314:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075101/real 1769075101] req@ffff9350050bf480 x1855008913056384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075101.170325:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913056384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075101.170333:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075101.170336:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075101.170341:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913056384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075101.170348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913056512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075101.170365:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075101.170375:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075101/real 1769075101] req@ffff9350050bfb80 x1855008913056512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075101.170385:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913056512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075101.170392:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075101.170396:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075101.170399:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913056512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075101.170422:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913056640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075101.170440:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075101.170452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075101/real 1769075101] req@ffff9350050bca80 x1855008913056640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075101.170463:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913056640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075101.170470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075101.170473:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075101.170476:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913056640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075101.170483:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913056768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075101.170502:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075101.170512:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075101/real 1769075101] req@ffff9350050bd500 x1855008913056768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075101.170525:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913056768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075101.170532:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075101.170535:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075101.170538:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913056768:192.168.204.121@tcp:8:kworker.0 00010000:00010000:1.0:1769075101.873298:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075102.897264:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075103.921241:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075104.945200:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075105.969257:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075106.289504:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913056896:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075106.289662:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075106.289778:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075106/real 1769075106] req@ffff935002a61c00 x1855008913056896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075106.289813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913056896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075106.289833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075106.289840:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075106.289846:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913056896:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075106.289859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913057024:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075106.289941:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075106.289964:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075106/real 1769075106] req@ffff935002a62a00 x1855008913057024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075106.289986:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913057024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075106.289999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075106.290006:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075106.290013:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913057024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075106.290023:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913057152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075106.290113:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075106.290135:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075106/real 1769075106] req@ffff935002a61880 x1855008913057152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075106.290154:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913057152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075106.290167:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075106.290173:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075106.290178:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913057152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075106.290228:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913057280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075106.290266:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075106.290284:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075106/real 1769075106] req@ffff935002a62680 x1855008913057280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075106.290304:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913057280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075106.290319:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075106.290325:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075106.290332:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913057280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075106.290344:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913057408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075106.290380:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075106.290399:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075106/real 1769075106] req@ffff935002a60000 x1855008913057408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075106.290417:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913057408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075106.290429:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075106.290434:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075106.290439:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913057408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075106.290449:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913057536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075106.290478:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075106.290720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075106/real 1769075106] req@ffff935002a61500 x1855008913057536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075106.290742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913057536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075106.290754:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075106.290759:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075106.290763:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913057536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075106.290773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913057664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075106.290804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075106.290822:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075106/real 1769075106] req@ffff935002a62300 x1855008913057664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075106.290840:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913057664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075106.290853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075106.290858:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075106.290862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913057664:192.168.204.121@tcp:8:kworker.0 00010000:00010000:1.0:1769075106.993259:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075108.017290:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075109.041264:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075110.065204:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075111.089254:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075112.113236:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075113.137332:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075114.161253:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075115.185225:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075115.697190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075116.209206:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075116.721175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075117.233199:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075117.745173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075118.257205:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075118.769156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075119.281245:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075119.793147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075120.305195:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075120.817178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075121.329304:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075121.649162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075121.649177:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075121.649182:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075121.649193:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 827 00000100:00080000:3.0:1769075121.649205:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075121.649211:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075121.649261:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913059584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075121.649285:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075121.649289:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075121.649290:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075121.649293:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 827 00000100:00080000:3.0:1769075121.649296:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075121.649298:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075121.649307:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075121.649315:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d880 x1855008913059712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075121.649322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075121.649325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075121.649326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075121.649329:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 827 00000100:00080000:3.0:1769075121.649330:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075121.649332:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075121.649335:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075121.649341:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913059840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075121.649347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075121.649350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075121.649351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075121.649354:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 827 00000100:00080000:3.0:1769075121.649356:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075121.649359:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075121.649362:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075121.649369:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d500 x1855008913059968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075121.649379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075121.649383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075121.649385:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075121.649388:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 827 00000100:00080000:3.0:1769075121.649390:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075121.649393:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075121.649396:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075121.649404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913060096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075121.649410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075121.649413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075121.649414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075121.649417:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 827 00000100:00080000:3.0:1769075121.649421:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075121.649423:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075121.649425:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075121.649431:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913060224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075121.649437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075121.649439:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075121.649441:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075121.649444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 827 00000100:00080000:3.0:1769075121.649446:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075121.649449:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075121.649452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075121.649460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913060352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075121.841173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075122.353201:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075122.865199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075123.377194:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075123.889136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075124.401222:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075124.721346:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075124.721392:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075124.721409:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075124.721453:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075124.721464:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075124.721474:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075124.721482:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075124.721489:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075124.721497:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075124.913166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075125.425327:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075125.937137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075126.449288:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075126.769158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075126.769171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075126.769176:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075126.769188:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 832 00000100:00080000:3.0:1769075126.769200:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075126.769209:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075126.769260:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913060480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075126.769294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075126.769299:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075126.769301:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075126.769306:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 832 00000100:00080000:3.0:1769075126.769311:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075126.769317:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075126.769328:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075126.769345:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913060608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075126.769361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075126.769365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075126.769378:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075126.769383:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 832 00000100:00080000:3.0:1769075126.769386:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075126.769391:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075126.769397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075126.769410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913060736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075126.769421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075126.769425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075126.769428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075126.769443:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 832 00000100:00080000:3.0:1769075126.769451:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075126.769458:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075126.769464:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075126.769479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913060864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075126.769492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075126.769503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075126.769508:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075126.769513:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 832 00000100:00080000:3.0:1769075126.769520:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075126.769540:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075126.769570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075126.769587:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913060992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075126.769608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075126.769613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075126.769616:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075126.769623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 832 00000100:00080000:3.0:1769075126.769632:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075126.769637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075126.769646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075126.769663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913061120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075126.769690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075126.769695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075126.769698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075126.769706:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 832 00000100:00080000:3.0:1769075126.769713:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075126.769723:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075126.769732:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075126.769751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913061248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075126.961179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075127.473257:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075127.985155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075128.497215:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075129.009131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075129.521312:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075130.033191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075130.545324:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075131.057200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075131.569320:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075131.889130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075131.889140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075131.889144:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075131.889154:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 837 00000100:00080000:3.0:1769075131.889165:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075131.889172:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075131.889212:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913061376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075131.889239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075131.889242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075131.889243:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075131.889247:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 837 00000100:00080000:3.0:1769075131.889249:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075131.889252:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075131.889261:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075131.889270:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d880 x1855008913061504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075131.889278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075131.889280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075131.889282:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075131.889285:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 837 00000100:00080000:3.0:1769075131.889287:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075131.889289:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075131.889292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075131.889298:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913061632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075131.889305:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075131.889307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075131.889309:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075131.889315:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 837 00000100:00080000:3.0:1769075131.889318:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075131.889323:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075131.889328:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075131.889334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913061760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075131.889340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075131.889343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075131.889344:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075131.889347:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 837 00000100:00080000:3.0:1769075131.889350:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075131.889353:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075131.889357:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075131.889364:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913061888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075131.889369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075131.889371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075131.889372:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075131.889375:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 837 00000100:00080000:3.0:1769075131.889379:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075131.889382:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075131.889384:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075131.889394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913062016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075131.889402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075131.889405:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075131.889406:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075131.889409:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 837 00000100:00080000:3.0:1769075131.889418:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075131.889421:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075131.889424:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075131.889434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d500 x1855008913062144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075132.081216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075132.593348:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075133.105112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075133.617371:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075134.129143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075134.641288:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075135.153104:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075135.665316:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075136.177139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075136.689197:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075137.009116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075137.009127:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075137.009131:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075137.009141:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 842 00000100:00080000:3.0:1769075137.009152:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075137.009158:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075137.009200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d500 x1855008913062272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075137.009239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075137.009242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075137.009243:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075137.009247:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 842 00000100:00080000:3.0:1769075137.009249:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075137.009253:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075137.009265:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075137.009281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913062400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075137.009310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075137.009330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075137.009332:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075137.009337:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 842 00000100:00080000:3.0:1769075137.009341:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075137.009345:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075137.009351:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075137.009371:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913062528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075137.009394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075137.009397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075137.009399:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075137.009402:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 842 00000100:00080000:3.0:1769075137.009408:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075137.009431:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075137.009435:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075137.009445:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913062656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075137.009452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075137.009455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075137.009456:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075137.009459:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 842 00000100:00080000:3.0:1769075137.009462:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075137.009465:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075137.009471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075137.009483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913062784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075137.009491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075137.009494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075137.009498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075137.009505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 842 00000100:00080000:3.0:1769075137.009509:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075137.009512:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075137.009520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075137.009531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d880 x1855008913062912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075137.009538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075137.009541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075137.009542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075137.009545:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 842 00000100:00080000:3.0:1769075137.009546:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075137.009548:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075137.009552:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075137.009561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913063040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075137.201160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075137.713229:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075138.225171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075138.737411:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075139.249260:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075139.761436:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075140.273183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075140.785264:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075141.297152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075141.809358:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075142.129175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075142.129188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075142.129193:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075142.129204:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 847 00000100:00080000:3.0:1769075142.129218:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075142.129226:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075142.129267:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913063168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075142.129311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075142.129315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075142.129317:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075142.129322:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 847 00000100:00080000:3.0:1769075142.129326:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075142.129331:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075142.129343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075142.129359:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913063296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075142.129370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075142.129374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075142.129376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075142.129380:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 847 00000100:00080000:3.0:1769075142.129385:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075142.129389:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075142.129394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075142.129412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913063424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075142.129428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075142.129432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075142.129435:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075142.129441:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 847 00000100:00080000:3.0:1769075142.129445:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075142.129449:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075142.129456:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075142.129469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913063552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075142.129481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075142.129485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075142.129488:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075142.129493:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 847 00000100:00080000:3.0:1769075142.129498:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075142.129503:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075142.129518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075142.129538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913063680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075142.129556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075142.129561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075142.129564:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075142.129568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 847 00000100:00080000:3.0:1769075142.129573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075142.129583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075142.129593:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075142.129615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913063808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075142.129674:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075142.129685:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075142.129688:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075142.129695:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 847 00000100:00080000:3.0:1769075142.129703:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075142.129718:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075142.129732:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075142.129754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913063936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075142.321147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075142.833239:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075143.345132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075143.857295:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075144.369169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075144.881387:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075145.393165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075145.905365:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075146.417197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075146.929225:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075147.249388:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913064064:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075147.249472:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075147.249578:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075147/real 1769075147] req@ffff9350050bed80 x1855008913064064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075147.249625:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bed80 x1855008913064064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075147.249642:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075147.249650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075147.249657:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913064064:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075147.249684:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913064192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075147.249706:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075147.249719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075147/real 1769075147] req@ffff9350050bd500 x1855008913064192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075147.249729:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913064192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075147.249735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075147.249738:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075147.249740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913064192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075147.249744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913064320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075147.249755:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075147.249761:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075147/real 1769075147] req@ffff9350050bca80 x1855008913064320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075147.249769:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913064320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075147.249774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075147.249775:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075147.249777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913064320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075147.249781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913064448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075147.249790:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075147.249795:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075147/real 1769075147] req@ffff9350050bfb80 x1855008913064448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075147.249803:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913064448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075147.249808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075147.249810:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075147.249811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913064448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075147.249815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913064576:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075147.249824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075147.249830:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075147/real 1769075147] req@ffff9350050bf480 x1855008913064576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075147.249837:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913064576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075147.249846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075147.249848:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075147.249989:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913064576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075147.249995:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913064704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075147.250016:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075147.250028:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075147/real 1769075147] req@ffff9350050bf100 x1855008913064704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075147.250038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913064704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075147.250044:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075147.250047:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075147.250111:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913064704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075147.250135:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913064832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075147.250154:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075147.250166:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075147/real 1769075147] req@ffff9350050bc000 x1855008913064832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075147.250175:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913064832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075147.250182:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075147.250184:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075147.250188:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913064832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075147.441166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075147.953364:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075148.465137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075148.977320:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075149.489227:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075150.001354:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075150.513206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075151.025322:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075151.537246:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075152.049431:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075152.369264:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075152.369280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075152.369286:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075152.369301:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 858 00000100:00080000:3.0:1769075152.369317:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075152.369327:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075152.369382:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d880 x1855008913064960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075152.369418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075152.369422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075152.369424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075152.369428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 858 00000100:00080000:3.0:1769075152.369432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075152.369437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075152.369448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075152.369462:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913065088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075152.369476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075152.369482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075152.369484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075152.369489:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 858 00000100:00080000:3.0:1769075152.369492:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075152.369495:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075152.369501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075152.369513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913065216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075152.369523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075152.369526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075152.369528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075152.369532:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 858 00000100:00080000:3.0:1769075152.369535:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075152.369538:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075152.369542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075152.369552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913065344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075152.369560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075152.369563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075152.369565:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075152.369575:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 858 00000100:00080000:3.0:1769075152.369582:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075152.369585:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075152.369590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075152.369601:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913065472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075152.369611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075152.369625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075152.369628:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075152.369635:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 858 00000100:00080000:3.0:1769075152.369644:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075152.369657:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075152.369667:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075152.369689:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d500 x1855008913065600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075152.369716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075152.369724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075152.369727:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075152.369732:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 858 00000100:00080000:3.0:1769075152.369737:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075152.369740:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075152.369746:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075152.369761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d880 x1855008913065728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075152.561159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075153.073371:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075153.585183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075154.097302:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075154.609231:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075155.121246:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075155.633186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075156.145391:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075156.657212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075157.169264:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075157.489190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075157.489207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075157.489215:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075157.489228:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 863 00000100:00080000:3.0:1769075157.489245:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075157.489255:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075157.489332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913065856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075157.489368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075157.489374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075157.489376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075157.489381:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 863 00000100:00080000:3.0:1769075157.489397:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075157.489404:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075157.489418:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075157.489440:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913065984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075157.489459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075157.489464:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075157.489467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075157.489473:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 863 00000100:00080000:3.0:1769075157.489480:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075157.489486:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075157.489494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075157.489511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913066112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075157.489521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075157.489529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075157.489533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075157.489540:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 863 00000100:00080000:3.0:1769075157.489544:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075157.489548:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075157.489554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075157.489566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913066240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075157.489574:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075157.489578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075157.489580:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075157.489584:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 863 00000100:00080000:3.0:1769075157.489588:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075157.489594:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075157.489602:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075157.489630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913066368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075157.489695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075157.489706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075157.489718:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075157.489723:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 863 00000100:00080000:3.0:1769075157.489728:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075157.489735:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075157.489747:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075157.489763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913066496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075157.489783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075157.489787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075157.489789:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075157.489798:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 863 00000100:00080000:3.0:1769075157.489805:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075157.489813:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075157.489819:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075157.489833:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913066624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075157.681204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075158.193325:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075158.705196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075159.217245:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075159.729125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075160.241424:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075160.753132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075161.265204:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075161.777140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075162.289271:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075162.801161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075163.313273:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075163.825187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075164.337307:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075164.849152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075165.361299:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075165.873131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075166.385404:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075166.897228:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075167.409315:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075167.729492:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913067648:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075167.729603:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075167.729691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075167/real 1769075167] req@ffff9350050bd180 x1855008913067648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075167.729776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd180 x1855008913067648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075167.729814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075167.729827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075167.729842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913067648:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075167.729904:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913067776:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075167.729957:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075167.729986:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075167/real 1769075167] req@ffff9350050bc000 x1855008913067776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075167.730015:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913067776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075167.730029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075167.730035:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075167.730040:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913067776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075167.730130:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913067904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075167.730171:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075167.730196:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075167/real 1769075167] req@ffff9350050bf100 x1855008913067904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075167.730222:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913067904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075167.730239:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075167.730246:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075167.730250:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913067904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075167.730259:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075167.730293:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075167.730314:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075167/real 1769075167] req@ffff9350050bf480 x1855008913068032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075167.730336:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913068032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075167.730351:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075167.730356:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075167.730363:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075167.730376:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913068160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075167.730412:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075167.730432:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075167/real 1769075167] req@ffff9350050bfb80 x1855008913068160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075167.730454:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913068160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075167.730470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075167.730476:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075167.730482:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913068160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075167.730494:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913068288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075167.730530:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075167.730554:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075167/real 1769075167] req@ffff9350050bca80 x1855008913068288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075167.730580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913068288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075167.730593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075167.730597:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075167.730601:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913068288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075167.730609:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913068416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075167.730637:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075167.730651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075167/real 1769075167] req@ffff9350050bd500 x1855008913068416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075167.730665:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913068416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075167.730683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075167.730687:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075167.730691:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913068416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075167.921155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075168.433405:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075168.945157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075169.457320:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075169.969185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075170.481362:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075170.993218:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075171.505310:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075172.017184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075172.529299:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075172.849431:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913068544:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075172.849541:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075172.849614:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075172/real 1769075172] req@ffff9350050bed80 x1855008913068544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075172.849667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bed80 x1855008913068544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075172.849692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075172.849711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075172.849726:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913068544:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075172.849751:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068672:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075172.849835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075172.849870:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075172/real 1769075172] req@ffff9350050bd500 x1855008913068672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075172.849930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913068672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075172.849955:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075172.849966:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075172.849977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075172.850035:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075172.850132:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075172.850159:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075172/real 1769075172] req@ffff9350050bca80 x1855008913068800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075172.850185:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913068800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075172.850202:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075172.850209:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075172.850216:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075172.850229:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075172.850264:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075172.850280:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075172/real 1769075172] req@ffff9350050bfb80 x1855008913068928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075172.850296:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913068928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075172.850307:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075172.850311:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075172.850316:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913068928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075172.850323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069056:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075172.850342:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075172.850353:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075172/real 1769075172] req@ffff9350050bf480 x1855008913069056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075172.850366:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913069056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075172.850376:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075172.850379:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075172.850383:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075172.850389:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075172.850406:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075172.850416:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075172/real 1769075172] req@ffff9350050bf100 x1855008913069184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075172.850431:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913069184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075172.850440:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075172.850443:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075172.850447:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075172.850453:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075172.850470:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075172.850481:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075172/real 1769075172] req@ffff9350050bc000 x1855008913069312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075172.850495:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913069312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075172.850504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075172.850507:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075172.850511:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075173.041172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075173.553251:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075174.065166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075174.577340:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075175.089235:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075175.601351:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075176.113180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075176.625367:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075177.137131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075177.649397:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075177.969544:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913069440:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075177.969649:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075177.969743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9350050bd180 x1855008913069440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075177.969795:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd180 x1855008913069440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075177.969880:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075177.969896:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075177.969904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913069440:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075177.969925:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913069568:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075177.969961:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075177.969985:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9350050bc000 x1855008913069568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075177.970037:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913069568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075177.970104:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075177.970131:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075177.970138:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913069568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075177.970190:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913069696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075177.970226:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075177.970251:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9350050bf100 x1855008913069696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075177.970276:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913069696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075177.970284:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075177.970288:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075177.970292:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913069696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075177.970299:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913069824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075177.970316:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075177.970325:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9350050bf480 x1855008913069824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075177.970337:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913069824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075177.970345:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075177.970348:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075177.970351:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913069824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075177.970355:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075177.970368:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075177.970376:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9350050bfb80 x1855008913069952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075177.970384:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913069952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075177.970391:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075177.970394:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075177.970397:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913069952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075177.970401:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913070080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075177.970414:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075177.970421:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9350050bca80 x1855008913070080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075177.970430:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913070080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075177.970437:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075177.970439:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075177.970443:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913070080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075177.970449:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913070208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075177.970462:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075177.970469:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9350050bd500 x1855008913070208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075177.970479:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913070208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075177.970486:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075177.970488:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075177.970491:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913070208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075178.161253:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075178.673276:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075179.185157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075179.697321:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075180.209132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075180.721342:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075181.233189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075181.745256:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075182.257156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075182.769260:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075183.089191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075183.089205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075183.089210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075183.089223:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 888 00000100:00080000:3.0:1769075183.089237:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075183.089249:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075183.089302:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913070336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075183.089351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075183.089358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075183.089361:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075183.089369:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 888 00000100:00080000:3.0:1769075183.089376:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075183.089383:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075183.089399:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075183.089445:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913070464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075183.089470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075183.089589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075183.089596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075183.089811:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 888 00000100:00080000:3.0:1769075183.089878:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075183.089894:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075183.089908:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075183.089939:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913070592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075183.089975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075183.089982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075183.089986:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075183.089994:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 888 00000100:00080000:3.0:1769075183.090005:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075183.090014:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075183.090028:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075183.090109:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913070720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075183.090132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075183.090139:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075183.090143:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075183.090155:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 888 00000100:00080000:3.0:1769075183.090165:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075183.090179:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075183.090195:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075183.090225:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913070848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075183.090250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075183.090260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075183.090265:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075183.090271:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 888 00000100:00080000:3.0:1769075183.090287:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075183.090296:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075183.090307:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075183.090332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913070976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075183.090370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075183.090378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075183.090385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075183.090394:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 888 00000100:00080000:3.0:1769075183.090401:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075183.090406:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075183.090415:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075183.090434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913071104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075183.281166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075183.793192:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075184.305171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075184.817240:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075185.329134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075185.841330:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075186.353154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075186.865287:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075187.377189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075187.889303:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075188.209241:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075188.209254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075188.209260:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075188.209272:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 893 00000100:00080000:3.0:1769075188.209285:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075188.209294:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075188.209367:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d880 x1855008913071232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075188.209408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075188.209413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075188.209416:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075188.209421:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 893 00000100:00080000:3.0:1769075188.209425:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075188.209431:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075188.209445:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075188.209459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d500 x1855008913071360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075188.209472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075188.209488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075188.209490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075188.209499:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 893 00000100:00080000:3.0:1769075188.209505:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075188.209509:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075188.209560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075188.209574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913071488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075188.209587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075188.209593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075188.209596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075188.209605:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 893 00000100:00080000:3.0:1769075188.209614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075188.209625:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075188.209631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075188.209646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913071616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075188.209656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075188.209661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075188.209662:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075188.209666:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 893 00000100:00080000:3.0:1769075188.209670:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075188.209674:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075188.209681:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075188.209699:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913071744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075188.209713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075188.209718:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075188.209721:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075188.209727:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 893 00000100:00080000:3.0:1769075188.209734:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075188.209739:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075188.209749:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075188.209772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913071872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075188.209789:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075188.209807:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075188.209810:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075188.209815:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 893 00000100:00080000:3.0:1769075188.209821:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075188.209835:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075188.209844:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075188.209861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913072000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075188.401141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075188.913319:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075189.425170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075189.937351:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075190.449119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075190.961279:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075191.281450:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075191.281552:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075191.281578:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075191.281645:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075191.281664:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075191.281678:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075191.281694:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075191.281714:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075191.281733:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075191.473148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075191.985241:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075192.497245:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075193.009239:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075193.329568:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913072128:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075193.329706:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075193.329790:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075193/real 1769075193] req@ffff935002a61c00 x1855008913072128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075193.329851:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913072128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075193.329888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075193.329908:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075193.329923:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913072128:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075193.329977:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913072256:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075193.330042:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075193.330145:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075193/real 1769075193] req@ffff935002a62a00 x1855008913072256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075193.330207:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913072256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075193.330262:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075193.330459:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075193.330467:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913072256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075193.330480:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913072384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075193.330528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075193.330555:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075193/real 1769075193] req@ffff935002a61880 x1855008913072384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075193.330574:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913072384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075193.330588:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075193.330594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075193.330600:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913072384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075193.330622:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913072512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075193.330656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075193.330675:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075193/real 1769075193] req@ffff935002a62680 x1855008913072512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075193.330708:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913072512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075193.330721:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075193.330725:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075193.330731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913072512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075193.330740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913072640:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075193.330768:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075193.330784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075193/real 1769075193] req@ffff935002a60000 x1855008913072640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075193.330801:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913072640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075193.330812:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075193.330816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075193.330820:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913072640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075193.330846:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913072768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075193.330874:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075193.330891:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075193/real 1769075193] req@ffff935002a61500 x1855008913072768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075193.330907:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913072768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075193.330917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075193.330921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075193.330926:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913072768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075193.330935:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913072896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075193.330961:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075193.330976:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075193/real 1769075193] req@ffff935002a62300 x1855008913072896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075193.330991:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913072896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075193.331001:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075193.331005:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075193.331009:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913072896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075193.521142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075194.033400:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075194.545145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075195.057198:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075195.569142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075196.081217:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075196.593212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075197.105285:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075197.617222:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075198.129404:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075198.449538:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913073024:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075198.449653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075198.449732:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075198/real 1769075198] req@ffff9350050bed80 x1855008913073024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075198.449792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bed80 x1855008913073024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075198.449852:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075198.449866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075198.449880:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913073024:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075198.449908:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913073152:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075198.449950:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075198.449981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075198/real 1769075198] req@ffff9350050bd500 x1855008913073152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075198.450017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913073152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075198.450035:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075198.450043:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075198.450107:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913073152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075198.450162:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913073280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075198.450210:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075198.450235:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075198/real 1769075198] req@ffff9350050bca80 x1855008913073280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075198.450253:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913073280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075198.450281:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075198.450286:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075198.450291:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913073280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075198.450299:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913073408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075198.450334:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075198.450373:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075198/real 1769075198] req@ffff9350050bfb80 x1855008913073408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075198.450390:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913073408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075198.450402:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075198.450406:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075198.450410:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913073408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075198.450424:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913073536:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075198.450450:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075198.450464:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075198/real 1769075198] req@ffff9350050bf480 x1855008913073536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075198.450478:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913073536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075198.450488:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075198.450492:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075198.450496:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913073536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075198.450503:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913073664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075198.450521:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075198.450532:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075198/real 1769075198] req@ffff9350050bf100 x1855008913073664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075198.450545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913073664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075198.450555:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075198.450558:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075198.450563:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913073664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075198.450570:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913073792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075198.450588:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075198.450598:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075198/real 1769075198] req@ffff9350050bc000 x1855008913073792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075198.450612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913073792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075198.450622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075198.450625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075198.450629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913073792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075198.641181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075199.153321:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075199.665256:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075200.177410:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075200.689163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075201.201414:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075201.713226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075202.225308:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075202.737166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075203.249332:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075203.569666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913073920:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075203.569764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075203.569849:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075203/real 1769075203] req@ffff935002a60a80 x1855008913073920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075203.569895:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913073920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075203.569932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075203.569945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075203.569952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913073920:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075203.569987:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074048:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075203.570026:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075203.570050:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075203/real 1769075203] req@ffff935002a61f80 x1855008913074048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075203.570127:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913074048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075203.570180:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075203.570188:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075203.570192:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075203.570201:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075203.570227:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075203.570238:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075203/real 1769075203] req@ffff935002a63100 x1855008913074176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075203.570251:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913074176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075203.570282:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075203.570304:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075203.570308:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075203.570316:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075203.570341:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075203.570354:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075203/real 1769075203] req@ffff935002a61180 x1855008913074304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075203.570367:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913074304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075203.570377:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075203.570380:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075203.570384:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075203.570390:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913074432:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075203.570406:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075203.570416:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075203/real 1769075203] req@ffff935002a62300 x1855008913074432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075203.570429:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913074432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075203.570437:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075203.570441:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075203.570444:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913074432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075203.570464:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913074560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075203.570481:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075203.570492:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075203/real 1769075203] req@ffff935002a61500 x1855008913074560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075203.570504:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913074560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075203.570513:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075203.570517:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075203.570521:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913074560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075203.570527:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913074688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075203.570561:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075203.570570:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075203/real 1769075203] req@ffff935002a60000 x1855008913074688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075203.570581:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913074688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075203.570589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075203.570593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075203.570596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913074688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075203.761241:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075204.273438:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075204.785158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075205.297344:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075205.809193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075206.321332:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075206.833144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075207.345223:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075207.857149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075208.369281:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075208.689686:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913074816:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075208.689802:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075208.689869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075208/real 1769075208] req@ffff935002a62680 x1855008913074816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075208.689914:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913074816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075208.689936:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075208.689952:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075208.689967:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913074816:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075208.689984:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074944:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075208.690031:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075208.690121:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075208/real 1769075208] req@ffff935002a61880 x1855008913074944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075208.690154:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913074944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075208.690193:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075208.690199:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075208.690203:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913074944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075208.690238:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075208.690279:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075208.690300:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075208/real 1769075208] req@ffff935002a62a00 x1855008913075072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075208.690325:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913075072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075208.690358:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075208.690366:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075208.690373:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075208.690383:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075208.690412:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075208.690445:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075208/real 1769075208] req@ffff935002a61c00 x1855008913075200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075208.690460:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913075200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075208.690471:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075208.690475:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075208.690479:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075208.690486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913075328:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075208.690507:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075208.690519:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075208/real 1769075208] req@ffff935002a60000 x1855008913075328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075208.690533:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913075328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075208.690542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075208.690545:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075208.690549:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913075328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075208.690556:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913075456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075208.690573:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075208.690584:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075208/real 1769075208] req@ffff935002a61500 x1855008913075456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075208.690598:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913075456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075208.690625:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075208.690629:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075208.690632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913075456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075208.690638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913075584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075208.690657:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075208.690667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075208/real 1769075208] req@ffff935002a62300 x1855008913075584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075208.690680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913075584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075208.690688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075208.690691:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075208.690695:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913075584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075208.881209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075209.393337:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075209.905198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075210.417372:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075210.929180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075211.441355:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075211.953258:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075212.465367:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075212.977228:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075213.489361:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075213.809536:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913075712:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075213.809644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075213.809718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075213/real 1769075213] req@ffff9350050bd180 x1855008913075712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075213.809812:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd180 x1855008913075712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075213.809853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075213.809878:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075213.809900:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913075712:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075213.809926:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075840:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075213.809979:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075213.810001:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075213/real 1769075213] req@ffff9350050bc000 x1855008913075840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075213.810019:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bc000 x1855008913075840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075213.810030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075213.810034:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075213.810038:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075840:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075213.810146:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075213.810187:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075213.810210:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075213/real 1769075213] req@ffff9350050bf100 x1855008913075968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075213.810237:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf100 x1855008913075968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075213.810268:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075213.810274:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075213.810281:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913075968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075213.810292:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913076096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075213.810322:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075213.810336:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075213/real 1769075213] req@ffff9350050bf480 x1855008913076096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075213.810351:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bf480 x1855008913076096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075213.810362:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075213.810366:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075213.810370:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913076096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075213.810377:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913076224:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075213.810396:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075213.810407:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075213/real 1769075213] req@ffff9350050bfb80 x1855008913076224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075213.810421:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bfb80 x1855008913076224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075213.810431:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075213.810434:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075213.810439:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913076224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075213.810446:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913076352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075213.810463:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075213.810474:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075213/real 1769075213] req@ffff9350050bca80 x1855008913076352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075213.810487:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bca80 x1855008913076352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075213.810497:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075213.810500:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075213.810504:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913076352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075213.810511:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913076480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075213.810529:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075213.810539:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075213/real 1769075213] req@ffff9350050bd500 x1855008913076480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075213.810552:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350050bd500 x1855008913076480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075213.810561:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075213.810565:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075213.810569:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350050bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913076480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075214.001230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075214.513401:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075215.025298:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075215.537283:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075216.049208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075216.561325:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075217.073244:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075217.585306:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075218.097223:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075218.609261:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075218.929207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.929220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075218.929232:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.929268:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 924 00000100:00080000:1.0:1769075218.929284:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.929295:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075218.929353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913076608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.929393:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.929399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.929403:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.929411:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 924 00000100:00080000:1.0:1769075218.929419:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.929426:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075218.929443:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.929474:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913076736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.929492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.929498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.929501:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.929508:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 924 00000100:00080000:1.0:1769075218.929515:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.929520:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075218.929528:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075218.929547:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913076864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.929560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.929564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075218.929567:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.929571:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 924 00000100:00080000:1.0:1769075218.929578:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.929584:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075218.929590:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075218.929610:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913076992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.929626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.929633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.929640:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.929651:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 924 00000100:00080000:1.0:1769075218.929659:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.929666:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075218.929675:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.929695:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913077120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.929715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.929730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.929735:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.929744:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 924 00000100:00080000:1.0:1769075218.929755:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.929766:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075218.929776:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.929821:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913077248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.929845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.929855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075218.929881:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.929894:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 924 00000100:00080000:1.0:1769075218.929907:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.929916:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075218.929928:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.929962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913077376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075219.121149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075219.633183:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075220.145198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075220.657310:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075221.169230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075221.681258:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075222.193188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075222.705285:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075223.217221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075223.729303:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075224.049226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075224.049236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075224.049240:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075224.049251:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 929 00000100:00080000:1.0:1769075224.049263:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075224.049276:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075224.049312:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913077504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075224.049335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075224.049338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075224.049339:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075224.049343:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 929 00000100:00080000:1.0:1769075224.049345:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075224.049348:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075224.049357:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075224.049367:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913077632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075224.049387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075224.049394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075224.049397:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075224.049401:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 929 00000100:00080000:1.0:1769075224.049403:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075224.049406:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075224.049409:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075224.049417:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913077760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075224.049424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075224.049427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075224.049428:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075224.049431:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 929 00000100:00080000:1.0:1769075224.049435:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075224.049437:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075224.049449:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075224.049457:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913077888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075224.049465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075224.049469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075224.049471:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075224.049475:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 929 00000100:00080000:1.0:1769075224.049479:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075224.049486:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075224.049495:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075224.049509:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913078016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075224.049536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075224.049539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075224.049541:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075224.049544:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 929 00000100:00080000:1.0:1769075224.049548:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075224.049555:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075224.049563:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075224.049575:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913078144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075224.049594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075224.049598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075224.049599:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075224.049603:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 929 00000100:00080000:1.0:1769075224.049608:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075224.049614:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075224.049618:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075224.049629:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913078272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075224.241142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075224.753325:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075225.265204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075225.777276:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075226.289126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075226.801218:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075227.313146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075227.825236:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075228.337212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075228.849260:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075229.169248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075229.169269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075229.169278:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075229.169297:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 934 00000100:00080000:3.0:1769075229.169317:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075229.169347:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075229.169407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913078400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075229.169462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075229.169470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075229.169475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075229.169485:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 934 00000100:00080000:3.0:1769075229.169494:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075229.169505:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075229.169526:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075229.169567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913078528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075229.169586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075229.169591:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075229.169594:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075229.169599:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 934 00000100:00080000:3.0:1769075229.169619:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075229.169630:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075229.169640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075229.169663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913078656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075229.169677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075229.169681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075229.169684:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075229.169690:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 934 00000100:00080000:3.0:1769075229.169696:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075229.169705:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075229.169712:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075229.169733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913078784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075229.169752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075229.169757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075229.169760:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075229.169770:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 934 00000100:00080000:3.0:1769075229.169774:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075229.169786:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075229.169794:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075229.169813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913078912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075229.169870:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075229.169880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075229.169889:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075229.169901:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 934 00000100:00080000:3.0:1769075229.169911:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075229.169917:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075229.169925:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075229.169941:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913079040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075229.169959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075229.169969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075229.169971:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075229.169978:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 934 00000100:00080000:3.0:1769075229.169983:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075229.169988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075229.169995:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075229.170015:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913079168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075229.361170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075229.873271:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075230.385159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075230.897334:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075231.409208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075231.921336:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075232.433187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075232.945179:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075233.457113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075233.969242:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075234.481220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075234.993365:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075235.505211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075236.017391:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075236.529170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075237.041207:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075237.553205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075238.065217:0:5555:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075238.577194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075239.089334:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075239.409167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075239.409181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075239.409187:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075239.409200:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 945 00000100:00080000:1.0:1769075239.409214:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075239.409222:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075239.409268:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913080192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075239.409301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075239.409306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075239.409308:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075239.409312:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 945 00000100:00080000:1.0:1769075239.409316:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075239.409320:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075239.409330:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075239.409342:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913080320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075239.409351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075239.409355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075239.409359:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075239.409366:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 945 00000100:00080000:1.0:1769075239.409368:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075239.409370:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075239.409374:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075239.409385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913080448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075239.409392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075239.409394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075239.409396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075239.409398:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 945 00000100:00080000:1.0:1769075239.409401:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075239.409404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075239.409406:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075239.409414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913080576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075239.409420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075239.409422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075239.409423:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075239.409426:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 945 00000100:00080000:1.0:1769075239.409428:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075239.409431:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075239.409437:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075239.409447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913080704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075239.409454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075239.409457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075239.409458:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075239.409461:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 945 00000100:00080000:1.0:1769075239.409463:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075239.409466:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075239.409469:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075239.409476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913080832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075239.409482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075239.409486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075239.409489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075239.409497:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 945 00000100:00080000:1.0:1769075239.409502:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075239.409507:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075239.409510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075239.409519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913080960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075239.601158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075240.113236:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075240.625161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075241.137291:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075241.137759:0:2362:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9350050bd180 x1855008913081088/t0(0) o103->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:17/18 lens 440/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075241.649189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075242.161290:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075242.673188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075243.185291:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075243.697117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075244.209207:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075244.529191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075244.529204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075244.529210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075244.529222:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 950 00000100:00080000:1.0:1769075244.529237:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075244.529247:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075244.529290:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913081216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075244.529321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075244.529325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075244.529327:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075244.529331:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 950 00000100:00080000:1.0:1769075244.529336:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075244.529341:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075244.529352:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075244.529369:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913081344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075244.529394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075244.529398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075244.529401:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075244.529408:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 950 00000100:00080000:1.0:1769075244.529412:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075244.529417:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075244.529423:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075244.529436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913081472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075244.529453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075244.529465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075244.529468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075244.529474:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 950 00000100:00080000:1.0:1769075244.529482:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075244.529488:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075244.529494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075244.529512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913081600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075244.529552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075244.529558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075244.529565:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075244.529592:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 950 00000100:00080000:1.0:1769075244.529602:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075244.529616:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075244.529629:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075244.529650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913081728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075244.529675:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075244.529681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075244.529684:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075244.529689:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 950 00000100:00080000:1.0:1769075244.529705:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075244.529714:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075244.529729:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075244.529751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913081856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075244.529791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075244.529796:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075244.529800:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075244.529808:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 950 00000100:00080000:1.0:1769075244.529817:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075244.529829:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075244.529844:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075244.529866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913081984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075244.721159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075245.745126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075246.257207:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075246.769150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075247.281245:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075247.793122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075248.305250:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075248.817131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075249.329292:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075249.841189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075250.353294:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075250.865181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075251.377308:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075251.889137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075252.401239:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075252.913142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075253.425258:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075253.937145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075254.449238:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075254.769173:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075254.769186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075254.769192:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075254.769202:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 960 00000100:00080000:1.0:1769075254.769217:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075254.769234:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075254.769271:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913083264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075254.769293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075254.769296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075254.769297:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075254.769301:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 960 00000100:00080000:1.0:1769075254.769304:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075254.769306:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075254.769314:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075254.769322:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913083392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075254.769329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075254.769332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075254.769334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075254.769336:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 960 00000100:00080000:1.0:1769075254.769339:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075254.769342:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075254.769346:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075254.769354:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913083520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075254.769365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075254.769368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075254.769370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075254.769372:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 960 00000100:00080000:1.0:1769075254.769374:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075254.769376:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075254.769380:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075254.769388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913083648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075254.769396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075254.769421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075254.769423:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075254.769426:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 960 00000100:00080000:1.0:1769075254.769428:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075254.769431:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075254.769435:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075254.769447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913083776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075254.769454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075254.769457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075254.769459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075254.769461:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 960 00000100:00080000:1.0:1769075254.769465:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075254.769468:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075254.769471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075254.769478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913083904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075254.769484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075254.769486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075254.769488:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075254.769490:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 960 00000100:00080000:1.0:1769075254.769494:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075254.769496:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075254.769499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075254.769505:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913084032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075254.961124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075255.473269:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075255.985161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075256.497209:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075257.009135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075257.521593:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075257.841303:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769075257.841382:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075257.841406:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075257.841458:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075257.841469:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075257.841480:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075257.841494:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075257.841510:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075257.841526:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075258.033189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075258.545217:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075259.057136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075259.569248:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075259.889174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075259.889185:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075259.889189:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075259.889199:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 965 00000100:00080000:1.0:1769075259.889211:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075259.889219:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075259.889263:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913084160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075259.889303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075259.889306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075259.889307:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075259.889321:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 965 00000100:00080000:1.0:1769075259.889325:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075259.889328:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075259.889339:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075259.889353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913084288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075259.889366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075259.889378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075259.889383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075259.889389:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 965 00000100:00080000:1.0:1769075259.889395:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075259.889400:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075259.889407:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075259.889424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913084416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075259.889435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075259.889439:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075259.889441:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075259.889447:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 965 00000100:00080000:1.0:1769075259.889451:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075259.889455:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075259.889462:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075259.889479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913084544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075259.889489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075259.889493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075259.889495:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075259.889500:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 965 00000100:00080000:1.0:1769075259.889504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075259.889509:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075259.889517:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075259.889535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913084672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075259.889549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075259.889553:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075259.889555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075259.889560:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 965 00000100:00080000:1.0:1769075259.889567:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075259.889575:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075259.889581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075259.889595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913084800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075259.889616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075259.889621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075259.889623:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075259.889629:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 965 00000100:00080000:1.0:1769075259.889633:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075259.889637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075259.889644:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075259.889661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913084928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075260.081204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075260.593269:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075261.105173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075261.617257:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075262.129182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075262.641223:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075263.153192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075263.665330:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075264.177171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075264.689337:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075265.009128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075265.009139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075265.009144:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075265.009152:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 970 00000100:00080000:1.0:1769075265.009175:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075265.009184:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075265.009246:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913085056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075265.009288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075265.009291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075265.009295:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075265.009300:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 970 00000100:00080000:1.0:1769075265.009306:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075265.009312:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075265.009324:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075265.009344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913085184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075265.009359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075265.009362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075265.009379:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075265.009385:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 970 00000100:00080000:1.0:1769075265.009389:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075265.009394:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075265.009400:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075265.009412:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913085312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075265.009421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075265.009424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075265.009426:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075265.009432:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 970 00000100:00080000:1.0:1769075265.009439:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075265.009442:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075265.009446:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075265.009458:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913085440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075265.009467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075265.009470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075265.009471:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075265.009474:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 970 00000100:00080000:1.0:1769075265.009477:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075265.009481:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075265.009486:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075265.009498:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913085568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075265.009536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075265.009543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075265.009547:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075265.009555:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 970 00000100:00080000:1.0:1769075265.009559:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075265.009566:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075265.009574:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075265.009588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913085696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075265.009595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075265.009598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075265.009600:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075265.009607:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 970 00000100:00080000:1.0:1769075265.009612:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075265.009616:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075265.009620:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075265.009630:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913085824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075265.201127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075265.713195:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075266.225171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075266.737353:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075267.249201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075267.761251:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075268.273217:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075268.785358:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075269.297210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075269.809333:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075270.321187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075270.833366:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075271.345177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075271.857297:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075272.369159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075272.881226:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075273.393122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075273.905229:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075274.417178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075274.929325:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075275.249234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075275.249245:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075275.249250:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075275.249259:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 980 00000100:00080000:1.0:1769075275.249271:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075275.249277:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075275.249329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913086848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075275.249355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075275.249358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075275.249360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075275.249364:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 980 00000100:00080000:1.0:1769075275.249367:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075275.249370:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075275.249380:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075275.249389:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913086976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075275.249397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075275.249400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075275.249402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075275.249405:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 980 00000100:00080000:1.0:1769075275.249406:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075275.249410:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075275.249415:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075275.249425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913087104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075275.249432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075275.249435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075275.249436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075275.249440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 980 00000100:00080000:1.0:1769075275.249443:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075275.249445:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075275.249448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075275.249456:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913087232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075275.249467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075275.249470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075275.249471:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075275.249474:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 980 00000100:00080000:1.0:1769075275.249476:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075275.249478:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075275.249482:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075275.249489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913087360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075275.249501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075275.249505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075275.249506:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075275.249510:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 980 00000100:00080000:1.0:1769075275.249513:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075275.249515:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075275.249518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075275.249526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913087488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075275.249531:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075275.249534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075275.249535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075275.249538:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 980 00000100:00080000:1.0:1769075275.249542:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075275.249548:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075275.249562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075275.249574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913087616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075275.441212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075275.953266:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075276.465132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075276.977274:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075277.489177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075278.001232:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075278.513193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075279.025288:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075279.537182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075280.049276:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075280.561120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075281.073279:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075281.585160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075282.097425:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075282.609160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075283.121326:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075283.633165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075284.145295:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075284.657118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075285.169245:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075285.681136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075286.193278:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075286.705190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075287.217230:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075287.729209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075288.241305:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075288.753165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075289.265260:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075289.777184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075290.289319:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075290.609354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075290.609370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075290.609376:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075290.609389:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 996 00000100:00080000:1.0:1769075290.609406:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075290.609417:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075290.609472:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913089536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075290.609516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075290.609522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075290.609536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075290.609542:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 996 00000100:00080000:1.0:1769075290.609548:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075290.609555:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075290.609569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075290.609586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913089664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075290.609606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075290.609614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075290.609628:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075290.609634:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 996 00000100:00080000:1.0:1769075290.609639:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075290.609644:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075290.609651:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075290.609668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913089792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075290.609680:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075290.609685:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075290.609688:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075290.609692:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 996 00000100:00080000:1.0:1769075290.609696:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075290.609700:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075290.609705:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075290.609718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913089920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075290.609747:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075290.609754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075290.609757:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075290.609764:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 996 00000100:00080000:1.0:1769075290.609768:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075290.609774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075290.609781:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075290.609795:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913090048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075290.609847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075290.609866:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075290.609872:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075290.609884:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 996 00000100:00080000:1.0:1769075290.609892:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075290.609904:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075290.609915:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075290.609946:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913090176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075290.609995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075290.610006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075290.610011:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075290.610021:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 996 00000100:00080000:1.0:1769075290.610032:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075290.610093:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075290.610102:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075290.610122:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913090304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075290.801224:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075291.313412:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075291.825163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075292.337318:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075292.849191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075293.361285:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075293.873209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075294.385258:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075294.897215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075295.409328:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075295.729181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075295.729192:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075295.729196:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075295.729214:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1001 00000100:00080000:1.0:1769075295.729228:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075295.729237:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075295.729291:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913090432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075295.729324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075295.729327:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075295.729330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075295.729333:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1001 00000100:00080000:1.0:1769075295.729337:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075295.729340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075295.729350:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075295.729362:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913090560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075295.729388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075295.729390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075295.729392:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075295.729396:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1001 00000100:00080000:1.0:1769075295.729399:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075295.729404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075295.729413:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075295.729426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913090688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075295.729434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075295.729437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075295.729439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075295.729442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1001 00000100:00080000:1.0:1769075295.729444:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075295.729447:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075295.729451:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075295.729458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913090816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075295.729476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075295.729479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075295.729481:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075295.729483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1001 00000100:00080000:1.0:1769075295.729487:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075295.729494:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075295.729500:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075295.729509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913090944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075295.729518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075295.729520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075295.729522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075295.729524:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1001 00000100:00080000:1.0:1769075295.729536:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075295.729538:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075295.729542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075295.729550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913091072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075295.729556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075295.729560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075295.729563:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075295.729568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1001 00000100:00080000:1.0:1769075295.729573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075295.729578:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075295.729584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075295.729599:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913091200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075295.921147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075296.433274:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075296.945215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075297.457265:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075297.969195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075298.481330:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 4 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769075298.481345:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216e800/0x410246667097e1b5 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400020000 nid: local remote: 0x48bf02eaaaf97bab expref: -99 pid: 22548 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769075298.481350:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1769075298.481397:0:5554:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769075298.481501:0:5554:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1769075298.481527:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff935008f0a400/0x410246667096e4ea lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400020000 nid: local remote: 0x48bf02eaaaf3ec82 expref: -99 pid: 20772 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769075298.481538:0:5554:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x1d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769075298.481561:0:5554:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x1d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1769075298.481571:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93500316ae00/0x410246667096e315 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->1052671) gid 0 flags: 0x800028400020000 nid: local remote: 0x48bf02eaaaf3e76c expref: -99 pid: 19002 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769075298.481581:0:5554:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x13:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769075298.481616:0:5554:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x13:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1769075298.481626:0:5554:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff935007715000/0x410246667096e028 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x48bf02eaaaf3df46 expref: -99 pid: 14434 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769075298.481635:0:5554:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769075298.481640:0:5554:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1769075298.481727:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216e800/0x410246667097e1b5 lrc: 1/0,0 mode: --/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf97bab expref: -99 pid: 22548 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481737:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff935008f0a400/0x410246667096e4ea lrc: 1/0,0 mode: --/PR res: [0x240000400:0x8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf3ec82 expref: -99 pid: 20772 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481743:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93500316ae00/0x410246667096e315 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->1052671) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf3e76c expref: -99 pid: 19002 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481756:0:5554:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff935007715000/0x410246667096e028 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x48bf02eaaaf3df46 expref: -99 pid: 14434 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481762:0:5554:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 4 locks packed 00010000:00010000:3.0:1769075298.481794:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93504216e800/0x410246667097e1b5 lrc: 0/0,0 mode: --/PR res: [0x240000400:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf97bab expref: -99 pid: 22548 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481816:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff935008f0a400/0x410246667096e4ea lrc: 0/0,0 mode: --/PR res: [0x240000400:0x8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf3ec82 expref: -99 pid: 20772 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481830:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff93500316ae00/0x410246667096e315 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->1052671) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf3e76c expref: -99 pid: 19002 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481844:0:5554:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff935006f53800 lock: ffff935007715000/0x410246667096e028 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x48bf02eaaaf3df46 expref: -99 pid: 14434 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075298.481862:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075298.481866:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075298.481871:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075298.993189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075299.505303:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075300.017133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075300.529328:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075300.849195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075300.849206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075300.849210:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075300.849221:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1006 00000100:00080000:1.0:1769075300.849232:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075300.849239:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075300.849296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913091456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075300.849328:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075300.849332:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075300.849334:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075300.849339:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1006 00000100:00080000:1.0:1769075300.849343:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075300.849347:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075300.849359:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075300.849398:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913091584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075300.849438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075300.849442:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075300.849446:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075300.849465:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1006 00000100:00080000:1.0:1769075300.849500:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075300.849505:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075300.849511:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075300.849530:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913091712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075300.849543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075300.849548:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075300.849550:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075300.849554:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1006 00000100:00080000:1.0:1769075300.849557:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075300.849561:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075300.849566:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075300.849578:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913091840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075300.849594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075300.849597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075300.849598:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075300.849605:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1006 00000100:00080000:1.0:1769075300.849611:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075300.849618:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075300.849622:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075300.849652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913091968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075300.849663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075300.849666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075300.849667:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075300.849671:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1006 00000100:00080000:1.0:1769075300.849673:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075300.849676:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075300.849681:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075300.849691:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913092096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075300.849700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075300.849703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075300.849704:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075300.849712:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1006 00000100:00080000:1.0:1769075300.849716:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075300.849720:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075300.849723:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075300.849735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913092224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075301.041192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075301.553243:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075302.065156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075302.577190:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075303.089138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075303.601322:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075304.113158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075304.625292:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075305.137191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075305.649256:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075305.969124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075305.969137:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075305.969142:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075305.969154:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1011 00000100:00080000:1.0:1769075305.969165:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075305.969172:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075305.969216:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913092352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075305.969244:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075305.969247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075305.969248:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075305.969251:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1011 00000100:00080000:1.0:1769075305.969254:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075305.969256:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075305.969266:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075305.969275:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913092480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075305.969283:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075305.969288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075305.969290:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075305.969298:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1011 00000100:00080000:1.0:1769075305.969301:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075305.969304:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075305.969307:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075305.969320:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913092608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075305.969330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075305.969333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075305.969334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075305.969337:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1011 00000100:00080000:1.0:1769075305.969339:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075305.969341:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075305.969344:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075305.969352:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913092736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075305.969358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075305.969361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075305.969362:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075305.969365:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1011 00000100:00080000:1.0:1769075305.969367:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075305.969372:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075305.969377:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075305.969385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913092864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075305.969392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075305.969394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075305.969395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075305.969398:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1011 00000100:00080000:1.0:1769075305.969401:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075305.969404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075305.969407:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075305.969414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913092992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075305.969420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075305.969423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075305.969424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075305.969428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1011 00000100:00080000:1.0:1769075305.969430:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075305.969433:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075305.969436:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075305.969447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913093120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075306.161187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075306.673272:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075307.185146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075307.697270:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075308.209183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075308.721270:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075309.233124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075309.745239:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075310.257147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075310.769379:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075311.281145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075311.793219:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075312.305211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075312.817199:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075313.329136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075313.841207:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075314.353143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075314.865257:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075315.377130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075315.889329:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075316.401172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075316.913216:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075317.425137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075317.937252:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075318.449160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075318.961225:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075319.473181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075319.985336:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075320.497232:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075321.009345:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075321.521195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075322.033386:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075322.545182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075323.057307:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075323.569187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075324.081284:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075324.401380:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075324.401422:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075324.401440:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075324.401479:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075324.401486:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075324.401493:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075324.401504:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075324.401516:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075324.401529:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075324.593161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075325.105378:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075325.617231:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075326.129281:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075326.449246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075326.449256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075326.449260:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075326.449271:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1032 00000100:00080000:1.0:1769075326.449281:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075326.449287:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075326.449324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913095936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075326.449366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075326.449370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075326.449372:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075326.449377:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1032 00000100:00080000:1.0:1769075326.449381:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075326.449385:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075326.449398:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075326.449415:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913096064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075326.449427:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075326.449432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075326.449435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075326.449444:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1032 00000100:00080000:1.0:1769075326.449448:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075326.449453:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075326.449458:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075326.449471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913096192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075326.449481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075326.449485:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075326.449487:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075326.449490:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1032 00000100:00080000:1.0:1769075326.449494:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075326.449498:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075326.449503:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075326.449518:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913096320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075326.449532:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075326.449536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075326.449538:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075326.449543:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1032 00000100:00080000:1.0:1769075326.449548:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075326.449552:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075326.449557:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075326.449569:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913096448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075326.449582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075326.449591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075326.449594:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075326.449603:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1032 00000100:00080000:1.0:1769075326.449608:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075326.449613:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075326.449618:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075326.449634:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913096576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075326.449651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075326.449659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075326.449667:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075326.449675:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1032 00000100:00080000:1.0:1769075326.449690:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075326.449694:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075326.449699:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075326.449714:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913096704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075326.641183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075327.153328:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075327.665192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075328.177246:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075328.689153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075329.201296:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075329.713270:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075330.225344:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075330.737166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075331.249225:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075331.761179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075332.273313:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075332.785196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075333.297260:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075333.809193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075334.321419:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075334.833184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075335.345376:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075335.857137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075336.369251:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075336.881127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075337.393254:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075337.905146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075338.417260:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075338.929144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075339.441201:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075339.953161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075340.465320:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075340.977125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075341.489221:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075341.809199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075341.809211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075341.809215:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075341.809225:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1047 00000100:00080000:1.0:1769075341.809236:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075341.809243:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075341.809285:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913098624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075341.809314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075341.809318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075341.809320:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075341.809324:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1047 00000100:00080000:1.0:1769075341.809328:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075341.809332:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075341.809344:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075341.809358:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913098752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075341.809369:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075341.809372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075341.809374:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075341.809378:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1047 00000100:00080000:1.0:1769075341.809382:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075341.809385:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075341.809390:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075341.809401:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913098880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075341.809409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075341.809413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075341.809414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075341.809417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1047 00000100:00080000:1.0:1769075341.809420:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075341.809435:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075341.809440:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075341.809450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913099008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075341.809458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075341.809461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075341.809463:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075341.809466:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1047 00000100:00080000:1.0:1769075341.809469:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075341.809473:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075341.809477:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075341.809485:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913099136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075341.809494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075341.809497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075341.809499:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075341.809503:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1047 00000100:00080000:1.0:1769075341.809506:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075341.809509:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075341.809513:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075341.809523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913099264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075341.809531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075341.809534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075341.809536:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075341.809540:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1047 00000100:00080000:1.0:1769075341.809543:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075341.809546:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075341.809550:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075341.809558:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913099392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075342.001121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075342.513266:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075343.025165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075343.537195:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075344.049183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075344.561260:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075345.073215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075345.585225:0:14430:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075346.097163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075346.609232:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075346.929342:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913099520:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075346.929414:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075346.929466:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff935002a61180 x1855008913099520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075346.929500:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913099520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075346.929530:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075346.929539:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075346.929549:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913099520:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075346.929579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913099648:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075346.929611:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075346.929627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff935002a63100 x1855008913099648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075346.929642:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913099648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075346.929654:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075346.929662:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075346.929667:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913099648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075346.929674:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913099776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075346.929690:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075346.929697:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff935002a61f80 x1855008913099776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075346.929704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913099776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075346.929709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075346.929711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075346.929713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913099776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075346.929716:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913099904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075346.929726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075346.929732:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff935002a60a80 x1855008913099904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075346.929739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913099904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075346.929745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075346.929747:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075346.929750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913099904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075346.929753:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075346.929762:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075346.929767:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff935002a62300 x1855008913100032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075346.929774:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913100032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075346.929778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075346.929781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075346.929783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075346.929796:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075346.929806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075346.929812:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff935002a61500 x1855008913100160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075346.929819:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913100160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075346.929824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075346.929826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075346.929828:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075346.929831:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075346.929841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075346.929846:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff935002a60000 x1855008913100288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075346.929882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913100288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075346.929887:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075346.929889:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075346.929891:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075347.121171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075347.633211:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075348.145124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075348.657290:0:14430:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 5 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769075348.657302:0:14430:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800028400020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769075348.657313:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075349.169188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075349.681212:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075350.193126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075350.705314:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075351.217177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075351.729242:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075352.049312:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913100416:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075352.049390:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075352.049440:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075352/real 1769075352] req@ffff935002a61c00 x1855008913100416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075352.049479:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913100416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075352.049499:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075352.049509:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075352.049516:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913100416:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075352.049543:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913100544:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075352.049571:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075352.049594:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075352/real 1769075352] req@ffff935002a62a00 x1855008913100544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075352.049607:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913100544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075352.049614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075352.049617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075352.049619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913100544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075352.049624:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913100672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075352.049637:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075352.049644:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075352/real 1769075352] req@ffff935002a61880 x1855008913100672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075352.049652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913100672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075352.049662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075352.049665:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075352.049667:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913100672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075352.049671:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913100800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075352.049680:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075352.049685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075352/real 1769075352] req@ffff935002a62680 x1855008913100800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075352.049693:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913100800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075352.049699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075352.049701:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075352.049703:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913100800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075352.049706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100928:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075352.049715:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075352.049721:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075352/real 1769075352] req@ffff935002a60000 x1855008913100928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075352.049729:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913100928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075352.049734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075352.049736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075352.049738:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913100928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075352.049743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075352.049761:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075352.049785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075352/real 1769075352] req@ffff935002a61500 x1855008913101056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075352.049799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913101056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075352.049809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075352.049812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075352.049816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075352.049836:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075352.049853:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075352.049861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075352/real 1769075352] req@ffff935002a62300 x1855008913101184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075352.049869:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913101184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075352.049874:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075352.049876:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075352.049879:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075352.241138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075352.753194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075352.753237:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075353.265173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075353.777197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075353.777306:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075354.289120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075354.801153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075354.801221:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075355.313167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075355.825184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075355.825275:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075356.337135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075356.849185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075356.849265:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075357.169212:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075357.169223:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075357.169227:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075357.169238:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1062 00000100:00080000:1.0:1769075357.169249:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075357.169255:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075357.169294:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913101312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075357.169326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075357.169329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075357.169331:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075357.169334:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1062 00000100:00080000:1.0:1769075357.169336:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075357.169340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075357.169349:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075357.169358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913101440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075357.169366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075357.169368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075357.169369:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075357.169372:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1062 00000100:00080000:1.0:1769075357.169374:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075357.169377:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075357.169381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075357.169382:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913101312:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769075357.169394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913101568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075357.169401:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075357.169404:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075357.169405:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075357.169408:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1062 00000100:00080000:1.0:1769075357.169410:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075357.169412:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075357.169415:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075357.169424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913101696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075357.169430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075357.169433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075357.169434:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075357.169437:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1062 00000100:00080000:1.0:1769075357.169439:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075357.169442:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075357.169444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075357.169451:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075357.169454:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913101824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075357.169469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075357.169472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075357.169473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075357.169476:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1062 00000100:00080000:1.0:1769075357.169478:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075357.169481:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075357.169484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075357.169492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913101952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075357.169499:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075357/real 1769075357] req@ffff9350062a4a80 x1855008913101312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075357.169501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075357.169507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075357.169510:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075357.169518:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1062 00000100:00080000:1.0:1769075357.169521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075357.169524:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075357.169527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075357.169534:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913101312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075357.169536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913102080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075357.169570:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075357.169578:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075357.169584:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913101312:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075357.169593:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913101440:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075357.169619:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075357.169633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075357/real 1769075357] req@ffff9350062a6680 x1855008913101440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075357.169647:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a6680 x1855008913101440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075357.169656:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075357.169660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075357.169664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913101440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075357.169723:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913101568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075357.169748:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075357.169761:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075357/real 1769075357] req@ffff9350062a5880 x1855008913101568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075357.169782:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913101568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075357.169790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075357.169793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075357.169795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913101568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075357.169800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913101696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075357.169815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075357.169823:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075357/real 1769075357] req@ffff9350062a5180 x1855008913101696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075357.169831:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913101696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075357.169836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075357.169838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075357.169840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913101696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075357.169843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075357.169853:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075357.169859:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075357/real 1769075357] req@ffff9350062a5c00 x1855008913101824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075357.169866:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913101824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075357.169871:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075357.169873:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075357.169875:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075357.169878:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075357.169891:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075357.169902:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075357/real 1769075357] req@ffff9350062a5500 x1855008913101952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075357.169915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913101952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075357.169924:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075357.169928:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075357.169931:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913101952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075357.169937:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075357.169954:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075357.169962:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075357/real 1769075357] req@ffff9350062a4380 x1855008913102080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075357.169969:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913102080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075357.169975:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075357.169977:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075357.169979:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075357.361204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075357.873195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075357.873281:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075358.385112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075358.897301:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075359.409169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075359.921217:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075360.433121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075360.945222:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075361.457119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075361.969246:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075362.289330:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913102208:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075362.289418:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075362.289477:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075362/real 1769075362] req@ffff935002a60a80 x1855008913102208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075362.289517:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913102208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075362.289527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075362.289531:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075362.289534:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913102208:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075362.289555:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913102336:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075362.289574:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075362.289584:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075362/real 1769075362] req@ffff935002a61f80 x1855008913102336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075362.289617:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913102336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075362.289624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075362.289627:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075362.289629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913102336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075362.289634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913102464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075362.289650:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075362.289658:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075362/real 1769075362] req@ffff935002a63100 x1855008913102464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075362.289667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913102464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075362.289672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075362.289675:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075362.289677:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913102464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075362.289681:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913102592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075362.289691:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075362.289696:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075362/real 1769075362] req@ffff935002a61180 x1855008913102592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075362.289704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913102592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075362.289715:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075362.289717:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075362.289719:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913102592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075362.289722:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102720:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075362.289731:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075362.289737:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075362/real 1769075362] req@ffff935002a62300 x1855008913102720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075362.289744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913102720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075362.289750:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075362.289752:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075362.289755:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075362.289761:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075362.289780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075362.289789:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075362/real 1769075362] req@ffff935002a61500 x1855008913102848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075362.289801:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913102848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075362.289808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075362.289811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075362.289814:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075362.289820:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075362.289837:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075362.289847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075362/real 1769075362] req@ffff935002a60000 x1855008913102976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075362.289859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913102976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075362.289866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075362.289869:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075362.289872:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913102976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075362.481155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075362.993215:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075363.505190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075364.017250:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075364.529220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075365.041214:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075365.553157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075366.065223:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075366.577277:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075367.089211:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075367.409154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075367.409165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075367.409169:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075367.409187:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1073 00000100:00080000:1.0:1769075367.409199:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075367.409205:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075367.409245:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913103104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075367.409270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075367.409273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075367.409274:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075367.409277:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1073 00000100:00080000:1.0:1769075367.409279:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075367.409283:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075367.409294:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075367.409308:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6d80 x1855008913103232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075367.409320:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075367.409323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913103104:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075367.409327:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075367.409331:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075367.409345:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1073 00000100:00080000:1.0:1769075367.409350:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075367.409353:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075367.409358:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075367.409370:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913103360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075367.409378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075367.409381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075367.409382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075367.409385:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1073 00000100:00080000:1.0:1769075367.409388:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075367.409393:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:3.0:1769075367.409394:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769075367.409396:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075367.409406:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913103488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075367.409413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075367.409417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075367.409418:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075367.409420:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1073 00000100:00080000:1.0:1769075367.409423:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075367.409425:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075367.409428:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075367.409436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913103616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075367.409443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075367.409446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769075367.409447:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075367/real 1769075367] req@ffff9350062a5f80 x1855008913103104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075367.409448:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075367.409454:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1073 00000100:00080000:1.0:1769075367.409458:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075367.409463:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075367.409469:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075367.409477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913103104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075367.409479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913103744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075367.409523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075367.409524:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075367.409526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075367.409529:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075367.409536:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1073 00000100:00080000:3.0:1769075367.409537:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769075367.409539:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075367.409545:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075367.409546:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913103104:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769075367.409550:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075367.409562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913103872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075367.409580:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913103232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075367.409601:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075367.409612:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075367/real 1769075367] req@ffff9350062a6d80 x1855008913103232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075367.409626:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a6d80 x1855008913103232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075367.409636:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075367.409639:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075367.409643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913103232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075367.409650:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913103360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075367.409671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075367.409683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075367/real 1769075367] req@ffff9350062a4380 x1855008913103360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075367.409692:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913103360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075367.409697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075367.409699:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075367.409702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913103360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075367.409706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913103488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075367.409719:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075367.409725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075367/real 1769075367] req@ffff9350062a5500 x1855008913103488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075367.409733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913103488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075367.409738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075367.409739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075367.409741:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913103488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075367.409745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913103616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075367.409755:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075367.409765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075367/real 1769075367] req@ffff9350062a5c00 x1855008913103616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075367.409777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913103616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075367.409785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075367.409788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075367.409792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913103616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075367.409798:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913103744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075367.409818:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075367.409826:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075367/real 1769075367] req@ffff9350062a5180 x1855008913103744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075367.409839:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913103744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075367.409845:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075367.409847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075367.409849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913103744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075367.409852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913103872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075367.409863:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075367.409868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075367/real 1769075367] req@ffff9350062a5880 x1855008913103872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075367.409875:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913103872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075367.409880:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075367.409881:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075367.409883:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913103872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075367.601165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075368.113235:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075368.625172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075369.137221:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075369.649134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075370.161284:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075370.673171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075371.185226:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075371.697141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075372.209231:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075372.529188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075372.529201:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075372.529207:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075372.529220:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1078 00000100:00080000:1.0:1769075372.529234:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075372.529242:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075372.529294:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913104000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075372.529329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075372.529333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075372.529335:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075372.529340:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1078 00000100:00080000:1.0:1769075372.529344:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075372.529348:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075372.529360:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075372.529374:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913104128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075372.529386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075372.529392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075372.529402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075372.529402:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913104000:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075372.529416:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1078 00000100:00080000:1.0:1769075372.529440:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075372.529463:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075372.529469:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075372.529480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913104256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075372.529489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075372.529491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075372.529493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075372.529497:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1078 00000100:00080000:1.0:1769075372.529501:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075372.529504:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075372.529507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1769075372.529536:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075372.529554:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913104384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075372.529574:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075372.529597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075372/real 1769075372] req@ffff9350062a6680 x1855008913104000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075372.529599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075372.529606:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075372.529622:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1078 00000100:00080000:1.0:1769075372.529629:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075372.529632:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a6680 x1855008913104000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075372.529640:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:3.0:1769075372.529645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769075372.529651:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769075372.529652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075372.529663:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913104000:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769075372.529677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913104512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075372.529687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075372.529690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075372.529692:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075372.529695:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913104128:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769075372.529705:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1078 00000100:00080000:1.0:1769075372.529712:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075372.529716:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075372.529721:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075372.529736:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913104640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075372.529738:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769075372.529755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075372.529758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075372.529761:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075372.529765:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1078 00000100:00080000:1.0:1769075372.529767:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075372.529772:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075372/real 1769075372] req@ffff9350062a4a80 x1855008913104128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075372.529773:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075372.529781:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075372.529799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913104768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075372.529807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913104128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075372.529819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075372.529823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075372.529825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913104128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075372.529834:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913104256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075372.529855:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075372.529862:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075372/real 1769075372] req@ffff9350062a5880 x1855008913104256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075372.529876:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913104256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075372.529881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075372.529883:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075372.529885:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913104256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075372.529889:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913104384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075372.529909:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075372.529916:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075372/real 1769075372] req@ffff9350062a5180 x1855008913104384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075372.529924:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913104384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075372.529930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075372.529932:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075372.529935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913104384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075372.529938:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913104512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075372.529949:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075372.529955:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075372/real 1769075372] req@ffff9350062a5c00 x1855008913104512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075372.529962:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913104512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075372.529972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075372.529975:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075372.529977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913104512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075372.529994:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913104640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075372.530004:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075372.530010:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075372/real 1769075372] req@ffff9350062a5500 x1855008913104640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075372.530018:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913104640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075372.530023:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075372.530025:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075372.530027:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913104640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075372.530030:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913104768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075372.530040:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075372.530046:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075372/real 1769075372] req@ffff9350062a4380 x1855008913104768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075372.530095:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913104768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075372.530104:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075372.530107:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075372.530111:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913104768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075372.721199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075373.233182:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075373.745132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075374.257239:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075374.769235:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075375.281243:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075375.793173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075376.305204:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075376.817154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075377.329296:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075377.649352:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913104896:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075377.649426:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075377.649474:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075377/real 1769075377] req@ffff9350103f9c00 x1855008913104896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075377.649498:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913104896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075377.649515:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075377.649523:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075377.649530:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913104896:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075377.649561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913105024:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075377.649579:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075377.649588:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075377/real 1769075377] req@ffff9350103f8700 x1855008913105024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075377.649597:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913105024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075377.649603:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075377.649605:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075377.649608:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913105024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075377.649612:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913105152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075377.649623:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075377.649629:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075377/real 1769075377] req@ffff9350103f9500 x1855008913105152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075377.649637:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913105152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075377.649641:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075377.649643:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075377.649645:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913105152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075377.649649:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913105280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075377.649659:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075377.649664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075377/real 1769075377] req@ffff9350103fad80 x1855008913105280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075377.649671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913105280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075377.649676:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075377.649678:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075377.649680:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913105280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075377.649683:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913105408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075377.649693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075377.649698:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075377/real 1769075377] req@ffff9350103fbb80 x1855008913105408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075377.649705:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913105408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075377.649710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075377.649712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075377.649714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913105408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075377.649717:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913105536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075377.649726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075377.649731:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075377/real 1769075377] req@ffff9350103fb800 x1855008913105536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075377.649738:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913105536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075377.649742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075377.649744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075377.649746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913105536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075377.649749:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913105664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075377.649758:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075377.649763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075377/real 1769075377] req@ffff9350103f8000 x1855008913105664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075377.649775:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8000 x1855008913105664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075377.649780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075377.649781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075377.649783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913105664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075377.841171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075378.353289:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075378.865173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075379.377288:0:14429:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075379.889155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075380.401196:0:5554:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075380.913181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075381.425211:0:14429:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 16 locks of 3 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769075381.425223:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa0a00/0x410246667097e248 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97d72 expref: -99 pid: 23704 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769075381.425239:0:5554:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 3 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769075381.425261:0:14429:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316a200/0x410246667097e25d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x48bf02eaaaf97dc6 expref: -99 pid: 23705 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769075381.425296:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356800/0x410246667097e21e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97d09 expref: -99 pid: 23704 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769075381.425318:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012357e00/0x410246667097e1fb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97c8b expref: -99 pid: 23125 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769075381.425340:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356a00/0x410246667097e1f4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97c76 expref: -99 pid: 23125 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769075381.425353:0:14429:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x28:0x0]: losing layout lock 00010000:00010000:1.0:1769075381.425382:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1800/0x410246667097e1ed lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x28:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97c4c expref: -99 pid: 23124 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769075381.425391:0:14429:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x28:0x0]: losing layout lock 00010000:00010000:1.0:1769075381.425398:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea4000/0x410246667097e1c3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97be3 expref: -99 pid: 23124 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769075381.425415:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316a400/0x410246667097e16f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97b0a expref: -99 pid: 22546 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769075381.425426:0:14429:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x25:0x0]: losing layout lock 00010000:00010000:1.0:1769075381.425440:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1c00/0x410246667097e168 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x25:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97ae0 expref: -99 pid: 22538 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769075381.425450:0:14429:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x25:0x0]: losing layout lock 00010000:00010000:1.0:1769075381.425458:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216fe00/0x410246667097e13e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x24:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97a77 expref: -99 pid: 22538 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769075381.425480:0:14429:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93504216e000/0x410246667097e129 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x21:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97a1c expref: -99 pid: 21970 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769075381.425534:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075381.425537:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075381.425539:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000002:00100000:1.0:1769075381.425546:0:14429:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9350103f8a80 x1855008913008640/t4294973361(4294973361) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'ls.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769075381.425560:0:14429:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:1.0:1769075381.425567:0:14429:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6d80 x1855008913105920/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:1.0:1769075381.425611:0:14429:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9350062a6d80 x1855008913105920/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1769075381.425637:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075381.937186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075382.449143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075382.449178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075382.769425:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913106176:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075382.769532:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075382.769605:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075382/real 1769075382] req@ffff935002a62680 x1855008913106176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075382.769639:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913106176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075382.769653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075382.769660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075382.769664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913106176:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075382.769674:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913106304:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075382.769703:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075382.769718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075382/real 1769075382] req@ffff935002a61880 x1855008913106304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075382.769732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913106304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075382.769742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075382.769746:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075382.769750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913106304:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075382.769780:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913106432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075382.769803:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075382.769817:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075382/real 1769075382] req@ffff935002a62a00 x1855008913106432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075382.769830:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913106432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075382.769840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075382.769843:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075382.769847:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913106432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075382.769854:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913106560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075382.769871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075382.769882:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075382/real 1769075382] req@ffff935002a61c00 x1855008913106560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075382.769895:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913106560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075382.769904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075382.769908:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075382.769912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913106560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075382.769919:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913106688:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075382.769939:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075382.769948:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075382/real 1769075382] req@ffff935002a60000 x1855008913106688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075382.769957:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913106688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075382.769963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075382.769965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075382.769968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913106688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075382.769971:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913106816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075382.769983:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075382.769989:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075382/real 1769075382] req@ffff935002a61500 x1855008913106816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075382.770002:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913106816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075382.770008:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075382.770010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075382.770011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913106816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075382.770015:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913106944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075382.770024:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075382.770031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075382/real 1769075382] req@ffff935002a62300 x1855008913106944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075382.770043:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913106944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075382.770085:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075382.770089:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075382.770093:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913106944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075382.961141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075383.473149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075383.473149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075383.473281:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075383.985162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075384.497131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075384.497131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075385.009181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075385.521202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075385.521216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075386.033182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075386.545146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075386.545168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075387.057237:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075387.569233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075387.569233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075387.889498:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913107072:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075387.889629:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913107200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075387.889694:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913107328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075387.889734:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913107456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075387.889773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913107584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075387.889800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913107712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075387.889824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913107840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075388.081157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075388.593132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075388.593132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075388.593658:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233f200/0x410246667096e473 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400020000 nid: local remote: 0x48bf02eaaaf3eb4e expref: -99 pid: 20191 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769075388.593709:0:25069:0:(vvp_io.c:1854:vvp_io_init()) [0x200000401:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769075388.593786:0:25069:0:(vvp_io.c:289:vvp_io_fini()) [0x200000401:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1769075388.593840:0:25069:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233f200/0x410246667096e473 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf3eb4e expref: -99 pid: 20191 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075388.593847:0:25069:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769075388.593871:0:25069:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff935006f53800 lock: ffff93501233f200/0x410246667096e473 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400020020 nid: local remote: 0x48bf02eaaaf3eb4e expref: -99 pid: 20191 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075388.593889:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075388.593892:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075388.593895:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075388.593930:0:2364:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff935002a7d880 x1855008913107968/t0(0) o103->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075389.105175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075389.617159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075389.617160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075390.129208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075390.641161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075390.641186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075390.961773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075390] req@ffff9350103fad80 x1855008913107328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075390.961839:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913107328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075390.961869:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075390.961878:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075390.961886:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913107328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075390.961899:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075390] req@ffff9350103f9500 x1855008913107456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075390.961922:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913107456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075390.961936:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075390.961941:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075390.961947:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913107456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075390.961959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075390] req@ffff9350103f8700 x1855008913107584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075390.961992:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913107584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075390.962007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075390.962013:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075390.962019:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913107584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075390.962031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075390] req@ffff9350103f9c00 x1855008913107712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075390.962094:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913107712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075390.962107:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075390.962138:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075390.962144:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913107712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075390.962156:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075390] req@ffff9350103fb800 x1855008913107072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075390.962179:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913107072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075390.962192:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075390.962197:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075390.962202:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913107072:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075390.962214:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075390] req@ffff9350103fbb80 x1855008913107200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075390.962234:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913107200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075390.962248:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075390.962254:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075390.962262:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913107200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075390.962327:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075390] req@ffff9350103fa680 x1855008913107840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075390.962354:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fa680 x1855008913107840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075390.962366:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075390.962372:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075390.962377:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913107840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075391.153138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075391.665198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075391.665198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075392.177184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075392.689130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075392.689142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075393.009360:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913108096:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075393.009446:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075393.009503:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075393/real 1769075393] req@ffff9350103fb100 x1855008913108096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075393.009558:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913108096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075393.009585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075393.009596:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075393.009604:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913108096:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075393.009656:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913108224:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075393.009707:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075393.009736:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075393/real 1769075393] req@ffff9350103f9180 x1855008913108224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075393.009777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913108224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075393.009792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075393.009797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075393.009802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913108224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075393.009811:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913108352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075393.009842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075393.009857:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075393/real 1769075393] req@ffff9350103fb480 x1855008913108352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075393.009872:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913108352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075393.009882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075393.009886:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075393.009890:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913108352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075393.009898:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913108480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075393.009922:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075393.009937:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075393/real 1769075393] req@ffff9350103f8380 x1855008913108480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075393.009953:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913108480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075393.009964:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075393.009968:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075393.009973:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913108480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075393.009981:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913108608:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075393.010007:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075393.010021:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075393/real 1769075393] req@ffff9350103faa00 x1855008913108608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075393.010038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913108608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075393.010712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075393.010723:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075393.010730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913108608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075393.010743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913108736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075393.010782:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075393.010800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075393/real 1769075393] req@ffff9350103f9f80 x1855008913108736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075393.010834:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913108736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075393.010846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075393.010850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075393.010854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913108736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075393.010889:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913108864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075393.010916:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075393.010930:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075393/real 1769075393] req@ffff9350103fb100 x1855008913108864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075393.010943:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913108864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075393.010952:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075393.010957:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075393.010961:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913108864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075393.201118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075393.713166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075393.713166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075394.225200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075394.737230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075394.737231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075395.249211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075395.761163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075395.761176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075396.273187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075396.785163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075396.785163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075397.297109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075397.809152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075397.809153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769075398.129172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075398.129182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075398.129186:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075398.129195:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1103 00000100:00080000:1.0:1769075398.129206:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075398.129214:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075398.129251:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913108992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075398.129276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075398.129279:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075398.129281:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075398.129284:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1103 00000100:00080000:1.0:1769075398.129287:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075398.129289:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075398.129296:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075398.129306:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913109120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075398.129314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075398.129317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075398.129319:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075398.129324:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1103 00000100:00100000:3.0:1769075398.129325:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913108992:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075398.129326:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075398.129333:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075398.129337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075398.129346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913109248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075398.129352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075398.129354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075398.129356:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075398.129358:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1103 00000100:00080000:1.0:1769075398.129361:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075398.129364:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075398.129366:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075398.129374:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913109376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075398.129379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075398.129381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075398.129383:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075398.129385:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1103 00000100:00080000:1.0:1769075398.129388:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075398.129392:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075398.129395:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075398.129402:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075398.129404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913109504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075398.129413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075398.129416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075398.129418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075398.129421:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1103 00000100:00080000:1.0:1769075398.129424:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075398.129427:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075398.129430:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075398.129438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913109632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075398.129443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075398.129445:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075398/real 1769075398] req@ffff9350062a5f80 x1855008913108992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075398.129446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075398.129450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075398.129457:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1103 00000100:00080000:1.0:1769075398.129464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075398.129467:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075398.129470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075398.129482:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913108992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075398.129483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913109760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075398.129511:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075398.129516:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075398.129520:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913108992:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075398.129541:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913109120:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075398.129557:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075398.129566:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075398/real 1769075398] req@ffff9350062a4380 x1855008913109120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075398.129574:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913109120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075398.129581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075398.129584:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075398.129586:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913109120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075398.129590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913109248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075398.129602:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075398.129608:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075398/real 1769075398] req@ffff9350062a5500 x1855008913109248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075398.129616:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913109248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075398.129622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075398.129623:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075398.129625:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913109248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075398.129629:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913109376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075398.129639:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075398.129644:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075398/real 1769075398] req@ffff9350062a5c00 x1855008913109376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075398.129652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913109376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075398.129658:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075398.129660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075398.129661:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913109376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075398.129665:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913109504:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075398.129674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075398.129680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075398/real 1769075398] req@ffff9350062a5180 x1855008913109504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075398.129687:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913109504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075398.129697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075398.129700:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075398.129702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913109504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075398.129705:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913109632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075398.129715:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075398.129720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075398/real 1769075398] req@ffff9350062a5880 x1855008913109632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075398.129727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913109632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075398.129733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075398.129735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075398.129737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913109632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075398.129740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913109760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075398.129749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075398.129755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075398/real 1769075398] req@ffff9350062a4a80 x1855008913109760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075398.129763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913109760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075398.129768:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075398.129769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075398.129772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913109760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075398.321154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075398.833204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075398.833212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075398.833331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075398.833804:0:2362:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9350103fa680 x1855008913109888/t0(0) o103->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:17/18 lens 384/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075399.345107:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075399.857153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075399.857156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075400.369139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075400.881192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075400.881192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075401.393166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075401.905127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075401.905156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075402.417121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075402.929196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075402.929218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075403.249326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913110016:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075403.249397:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075403.249445:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075403/real 1769075403] req@ffff9350103fbb80 x1855008913110016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075403.249463:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913110016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075403.249476:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075403.249481:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075403.249485:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913110016:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075403.249512:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913110144:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075403.249533:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075403.249546:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075403/real 1769075403] req@ffff9350103fb800 x1855008913110144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075403.249566:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913110144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075403.249575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075403.249578:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075403.249581:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913110144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075403.249587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913110272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075403.249609:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075403.249619:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075403/real 1769075403] req@ffff9350103f9c00 x1855008913110272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075403.249628:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913110272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075403.249633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075403.249636:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075403.249638:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913110272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075403.249641:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913110400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075403.249653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075403.249659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075403/real 1769075403] req@ffff9350103f8700 x1855008913110400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075403.249666:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913110400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075403.249675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075403.249677:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075403.249680:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913110400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075403.249683:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913110528:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075403.249692:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075403.249698:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075403/real 1769075403] req@ffff9350103f9500 x1855008913110528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075403.249704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913110528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075403.249710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075403.249712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075403.249714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913110528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075403.249727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913110656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075403.249736:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075403.249742:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075403/real 1769075403] req@ffff9350103fad80 x1855008913110656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075403.249748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913110656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075403.249754:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075403.249755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075403.249758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913110656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075403.249761:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913110784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075403.249770:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075403.249776:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075403/real 1769075403] req@ffff9350103fbb80 x1855008913110784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075403.249782:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913110784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075403.249788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075403.249789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075403.249791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913110784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075403.441125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075403.953150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075403.953153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075404.465121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075404.977216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075404.977216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075405.489163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075406.001186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075406.001190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075406.001386:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075406.513139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075407.025153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075407.025153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075407.537146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075408.049107:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075408.049154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075408.049197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075408.369399:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913110912:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075408.369469:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075408.369513:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075408/real 1769075408] req@ffff9350103fb100 x1855008913110912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075408.369539:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913110912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075408.369563:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075408.369574:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075408.369580:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913110912:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075408.369610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111040:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075408.369635:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075408.369650:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075408/real 1769075408] req@ffff9350103f9f80 x1855008913111040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075408.369661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913111040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075408.369668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075408.369671:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075408.369673:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075408.369678:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075408.369690:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075408.369698:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075408/real 1769075408] req@ffff9350103faa00 x1855008913111168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075408.369710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913111168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075408.369725:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075408.369728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075408.369732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075408.369738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075408.369769:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075408.369780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075408/real 1769075408] req@ffff9350103f8380 x1855008913111296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075408.369793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913111296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075408.369801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075408.369803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075408.369806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075408.369810:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913111424:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075408.369841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075408.369850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075408/real 1769075408] req@ffff9350103fb480 x1855008913111424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075408.369858:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913111424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075408.369873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075408.369875:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075408.369878:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913111424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075408.369882:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913111552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075408.369900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075408.369911:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075408/real 1769075408] req@ffff9350103f9180 x1855008913111552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075408.369922:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913111552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075408.369930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075408.369933:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075408.369937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913111552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075408.369958:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913111680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075408.369976:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075408.369986:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075408/real 1769075408] req@ffff9350103fb100 x1855008913111680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075408.369997:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913111680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075408.370005:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075408.370008:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075408.370011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913111680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075408.561143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075409.073219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075409.073226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075409.585202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075410.097167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075410.097167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075410.609139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075411.121150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075411.121204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075411.633146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075412.145204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075412.145214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075412.145284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075412.657136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075413.169194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075413.169194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075413.489568:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913111808:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075413.489668:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075413.489733:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075413/real 1769075413] req@ffff935002a61180 x1855008913111808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075413.489793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913111808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075413.489841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075413.489859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075413.489878:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913111808:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075413.489897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075413.489948:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075413.489966:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075413/real 1769075413] req@ffff935002a63100 x1855008913111936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075413.489981:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913111936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075413.489991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075413.489996:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075413.489999:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913111936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075413.490029:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075413.490083:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075413.490097:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075413/real 1769075413] req@ffff935002a61f80 x1855008913112064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075413.490111:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913112064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075413.490120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075413.490124:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075413.490127:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075413.490133:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075413.490170:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075413.490180:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075413/real 1769075413] req@ffff935002a60a80 x1855008913112192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075413.490192:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913112192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075413.490200:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075413.490203:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075413.490206:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075413.490211:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913112320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075413.490227:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075413.490236:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075413/real 1769075413] req@ffff935002a62300 x1855008913112320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075413.490247:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913112320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075413.490255:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075413.490258:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075413.490261:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913112320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075413.490266:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913112448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075413.490280:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075413.490289:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075413/real 1769075413] req@ffff935002a61500 x1855008913112448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075413.490300:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913112448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075413.490308:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075413.490310:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075413.490313:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913112448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075413.490318:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913112576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075413.490332:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075413.490341:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075413/real 1769075413] req@ffff935002a60000 x1855008913112576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075413.490359:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913112576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075413.490367:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075413.490369:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075413.490372:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913112576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075413.681155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075414.193120:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075414.193121:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075414.193169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075414.705132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075415.217184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075415.729137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075416.241163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075416.241256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075416.753164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075417.265216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075417.777177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075418.289225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075418.289326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075418.609332:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913112704:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075418.609420:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075418.609476:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075418/real 1769075418] req@ffff9350103fbb80 x1855008913112704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075418.609522:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913112704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075418.609547:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075418.609557:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075418.609567:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913112704:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075418.609606:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075418.609669:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075418.609687:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075418/real 1769075418] req@ffff9350103fad80 x1855008913112832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075418.609702:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913112832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075418.609711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075418.609715:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075418.609719:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075418.609726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075418.609749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075418.609761:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075418/real 1769075418] req@ffff9350103f9500 x1855008913112960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075418.609773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913112960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075418.609782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075418.609785:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075418.609789:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913112960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075418.609794:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075418.609813:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075418.609824:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075418/real 1769075418] req@ffff9350103f8700 x1855008913113088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075418.609835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913113088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075418.609844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075418.609846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075418.609849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075418.609855:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913113216:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075418.609871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075418.609880:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075418/real 1769075418] req@ffff9350103f9c00 x1855008913113216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075418.609972:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913113216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075418.609981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075418.609984:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075418.609987:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913113216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075418.609994:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913113344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075418.610016:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075418.610028:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075418/real 1769075418] req@ffff9350103fb800 x1855008913113344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075418.610041:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913113344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075418.610091:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075418.610095:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075418.610099:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913113344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075418.610122:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913113472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075418.610145:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075418.610156:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075418/real 1769075418] req@ffff9350103fbb80 x1855008913113472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075418.610167:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913113472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075418.610174:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075418.610177:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075418.610181:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913113472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075418.801127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075419.313143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075419.825171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075420.337261:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075420.337399:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075420.849177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075421.361158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075421.873158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075422.385241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075422.385351:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075422.897285:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075423.409310:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075423.729471:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913113600:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075423.729607:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075423.729675:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075423/real 1769075423] req@ffff935002a61c00 x1855008913113600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075423.729708:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913113600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075423.729759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075423.729778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075423.729788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913113600:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075423.729854:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075423.729920:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075423.729959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075423/real 1769075423] req@ffff935002a62a00 x1855008913113728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075423.729981:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913113728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075423.729992:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075423.730014:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075423.730019:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075423.730028:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075423.730058:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075423.730124:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075423/real 1769075423] req@ffff935002a61880 x1855008913113856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075423.730142:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913113856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075423.730150:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075423.730153:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075423.730156:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075423.730161:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075423.730179:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075423.730189:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075423/real 1769075423] req@ffff935002a62680 x1855008913113984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075423.730206:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913113984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075423.730217:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075423.730221:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075423.730226:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913113984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075423.730234:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913114112:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075423.730261:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075423.730277:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075423/real 1769075423] req@ffff935002a60000 x1855008913114112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075423.730290:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913114112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075423.730303:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075423.730306:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075423.730310:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913114112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075423.730315:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913114240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075423.730333:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075423.730342:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075423/real 1769075423] req@ffff935002a61500 x1855008913114240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075423.730357:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913114240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075423.730368:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075423.730372:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075423.730377:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913114240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075423.730385:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913114368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075423.730411:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075423.730426:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075423/real 1769075423] req@ffff935002a62300 x1855008913114368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075423.730440:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913114368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075423.730447:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075423.730450:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075423.730453:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913114368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075423.921129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075424.433183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075424.433358:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075424.945162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075425.457145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075425.969140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075426.481203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075426.481339:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075426.993142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075427.505247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075427.505369:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075428.017156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075428.529166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075428.529316:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075428.849520:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913114496:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075428.849672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075428.849733:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff935002a60a80 x1855008913114496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075428.849755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913114496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075428.849766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075428.849771:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075428.849775:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913114496:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075428.849784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913114624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075428.849804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075428.849816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff935002a61f80 x1855008913114624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075428.849836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913114624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075428.849846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075428.849849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075428.849852:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913114624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075428.849858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913114752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075428.849891:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075428.849900:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff935002a63100 x1855008913114752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075428.849910:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913114752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075428.849917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075428.849921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075428.849925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913114752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075428.849960:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913114880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075428.849987:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075428.850002:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff935002a61180 x1855008913114880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075428.850019:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913114880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075428.850029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075428.850034:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075428.850039:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913114880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075428.850046:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115008:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075428.850105:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075428.850119:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff935002a62300 x1855008913115008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075428.850130:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913115008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075428.850137:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075428.850140:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075428.850143:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075428.850148:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075428.850164:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075428.850172:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff935002a61500 x1855008913115136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075428.850181:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913115136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075428.850187:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075428.850190:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075428.850193:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075428.850198:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075428.850211:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075428.850218:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff935002a60000 x1855008913115264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075428.850227:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913115264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075428.850250:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075428.850253:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075428.850255:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075429.041125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075429.553127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075429.553199:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075430.065147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075430.577123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075430.577280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075431.089173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075431.601147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075431.601210:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075432.113190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075432.625140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075432.625189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075433.137248:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075433.649174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075433.649281:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075433.969429:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913115392:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075433.969533:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075433.969597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff935002a62680 x1855008913115392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.969639:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913115392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.969716:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.969735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075433.969749:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913115392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075433.969803:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913115520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075433.969850:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075433.969878:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff935002a61880 x1855008913115520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.969893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913115520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.969904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.969908:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075433.969912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913115520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075433.969920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913115648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075433.969941:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075433.969949:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff935002a62a00 x1855008913115648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.969959:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913115648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.969969:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.969971:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075433.969974:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913115648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075433.969978:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913115776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075433.969993:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075433.970003:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff935002a61c00 x1855008913115776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.970015:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913115776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.970025:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.970029:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075433.970032:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913115776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075433.970040:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115904:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075433.970103:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075433.970112:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff935002a60000 x1855008913115904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.970120:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913115904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.970126:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.970129:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075433.970131:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913115904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075433.970148:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075433.970160:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075433.970166:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff935002a61500 x1855008913116032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.970173:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913116032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.970179:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.970180:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075433.970182:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075433.970197:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075433.970207:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075433.970214:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff935002a62300 x1855008913116160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.970229:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913116160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.970235:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.970237:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075433.970240:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075434.161127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075434.673138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075434.673201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075435.185182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075435.697169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075435.697241:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075436.209117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075436.721147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075436.721311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075437.233116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075437.745159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075437.745343:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075438.257176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075438.769126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075438.769265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075439.089436:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913116288:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075439.089532:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075439.089601:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075439/real 1769075439] req@ffff935002a61180 x1855008913116288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075439.089638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913116288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075439.089671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075439.089682:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075439.089694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913116288:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075439.089715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913116416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075439.089752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075439.089770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075439/real 1769075439] req@ffff935002a63100 x1855008913116416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075439.089787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913116416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075439.089807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075439.089811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075439.089816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913116416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075439.089850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913116544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075439.089879:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075439.089913:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075439/real 1769075439] req@ffff935002a61f80 x1855008913116544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075439.089929:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913116544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075439.089940:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075439.089944:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075439.089949:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913116544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075439.089956:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913116672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075439.089982:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075439.089996:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075439/real 1769075439] req@ffff935002a60a80 x1855008913116672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075439.090012:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913116672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075439.090022:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075439.090026:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075439.090031:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913116672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075439.090038:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116800:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075439.090131:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075439.090149:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075439/real 1769075439] req@ffff935002a62300 x1855008913116800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075439.090184:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913116800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075439.090198:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075439.090203:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075439.090208:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075439.090235:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075439.090257:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075439.090270:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075439/real 1769075439] req@ffff935002a61500 x1855008913116928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075439.090284:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913116928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075439.090293:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075439.090296:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075439.090300:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913116928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075439.090327:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075439.090374:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075439.090396:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075439/real 1769075439] req@ffff935002a60000 x1855008913117056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075439.090422:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913117056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075439.090434:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075439.090438:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075439.090442:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075439.281183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075439.793146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075439.793321:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075440.305348:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075440.817165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075440.817337:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075441.329189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075441.841182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075441.841335:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075442.353269:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075442.865136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075442.865203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075443.377139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075443.889209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075443.889440:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075444.209355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075444.209372:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075444.209378:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075444.209412:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1149 00000100:00080000:1.0:1769075444.209427:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075444.209435:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075444.209495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913117184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075444.209549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075444.209556:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075444.209561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075444.209570:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1149 00000100:00080000:1.0:1769075444.209578:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075444.209586:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075444.209620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075444.209644:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913117312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075444.209660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075444.209666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075444.209673:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075444.209675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913117184:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075444.209699:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1149 00000100:00080000:1.0:1769075444.209709:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075444.209715:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075444.209721:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075444.209733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913117440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075444.209743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075444.209747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075444.209750:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075444.209756:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1149 00000100:00080000:1.0:1769075444.209760:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075444.209764:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075444.209769:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075444.209785:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913117568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075444.209797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075444.209801:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075444.209804:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075444.209808:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1149 00000100:00080000:1.0:1769075444.209813:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075444.209816:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075444.209822:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075444.209824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075444.209840:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913117696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075444.209853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075444.209858:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075444.209861:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075444.209865:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1149 00000100:00080000:1.0:1769075444.209868:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075444.209872:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075444.209878:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075444.209892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913117824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075444.209906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075444.209907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075444/real 1769075444] req@ffff9350062a6680 x1855008913117184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075444.209920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075444.209926:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075444.209939:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1149 00000100:00080000:1.0:1769075444.209949:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075444.209954:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075444.209961:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075444.209973:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a6680 x1855008913117184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075444.209977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913117952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075444.210033:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075444.210042:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075444.210113:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913117184:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075444.210147:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913117312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075444.210182:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075444.210199:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075444/real 1769075444] req@ffff9350062a4a80 x1855008913117312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075444.210215:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913117312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075444.210258:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075444.210262:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075444.210267:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913117312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075444.210276:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913117440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075444.210303:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075444.210317:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075444/real 1769075444] req@ffff9350062a5880 x1855008913117440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075444.210332:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913117440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075444.210341:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075444.210344:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075444.210348:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913117440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075444.210354:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913117568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075444.210372:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075444.210382:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075444/real 1769075444] req@ffff9350062a5180 x1855008913117568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075444.210395:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913117568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075444.210404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075444.210407:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075444.210411:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913117568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075444.210417:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117696:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075444.210453:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075444.210465:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075444/real 1769075444] req@ffff9350062a5c00 x1855008913117696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075444.210479:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913117696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075444.210489:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075444.210509:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075444.210513:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117696:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075444.210519:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075444.210537:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075444.210548:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075444/real 1769075444] req@ffff9350062a5500 x1855008913117824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075444.210569:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913117824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075444.210578:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075444.210581:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075444.210584:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075444.210590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075444.210606:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075444.210622:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075444/real 1769075444] req@ffff9350062a4380 x1855008913117952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075444.210644:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913117952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075444.210658:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075444.210664:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075444.210670:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913117952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075444.401163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075444.913146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075444.913203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075445.425221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075445.937172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075445.937305:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075446.449157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075446.961135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075446.961201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075447.473126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075447.985135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075447.985255:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075448.497199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075449.009151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075449.009250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075449.329243:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075449.329256:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075449.329261:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075449.329271:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1155 00000100:00080000:1.0:1769075449.329285:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075449.329293:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075449.329336:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913118080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075449.329370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075449.329374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075449.329376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075449.329380:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1155 00000100:00080000:1.0:1769075449.329384:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075449.329387:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075449.329398:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075449.329413:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913118208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075449.329448:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913118080:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075449.329450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075449.329468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075449.329470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075449.329475:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1155 00000100:00080000:1.0:1769075449.329479:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075449.329483:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075449.329490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075449.329499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913118336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075449.329505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075449.329508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075449.329510:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075449.329513:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1155 00000100:00080000:1.0:1769075449.329516:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075449.329518:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075449.329521:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075449.329528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913118464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075449.329535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075449.329537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075449.329538:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075449.329546:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1155 00000800:00000100:3.0:1769075449.329547:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769075449.329553:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075449.329560:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075449.329566:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075449.329579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913118592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075449.329586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075449.329590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075449.329593:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075449.329598:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1155 00000100:00080000:1.0:1769075449.329602:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075449.329604:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075449/real 1769075449] req@ffff9350062a5f80 x1855008913118080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075449.329608:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075449.329621:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075449.329634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913118720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075449.329640:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913118080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075449.329652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075449.329670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075449.329677:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075449.329689:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1155 00000100:00080000:3.0:1769075449.329689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075449.329699:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075449.329706:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:3.0:1769075449.329707:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769075449.329711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075449.329721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913118848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075449.329725:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913118080:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075449.329774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913118208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075449.329797:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075449.329811:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075449/real 1769075449] req@ffff9350062a4380 x1855008913118208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075449.329826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913118208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075449.329836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075449.329840:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075449.329844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913118208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075449.329852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913118336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075449.329876:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075449.329890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075449/real 1769075449] req@ffff9350062a5500 x1855008913118336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075449.329904:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913118336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075449.329914:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075449.329918:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075449.329921:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913118336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075449.329929:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913118464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075449.329946:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075449.329954:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075449/real 1769075449] req@ffff9350062a5c00 x1855008913118464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075449.329962:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913118464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075449.329968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075449.329970:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075449.329972:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913118464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075449.329976:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913118592:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075449.329986:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075449.329992:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075449/real 1769075449] req@ffff9350062a5180 x1855008913118592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075449.330004:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913118592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075449.330009:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075449.330011:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075449.330013:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913118592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075449.330016:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913118720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075449.330026:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075449.330032:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075449/real 1769075449] req@ffff9350062a5880 x1855008913118720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075449.330058:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913118720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075449.330095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075449.330098:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075449.330101:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913118720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075449.330106:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913118848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075449.330124:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075449.330165:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075449/real 1769075449] req@ffff9350062a4a80 x1855008913118848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075449.330175:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913118848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075449.330184:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075449.330187:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075449.330189:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913118848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075449.521150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075450.033130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075450.033236:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075450.545142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075451.057194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075451.057278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075451.569159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075452.081190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075452.081307:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075452.593188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075453.105200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075453.105305:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075453.617136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075454.129142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075454.129239:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075454.449399:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913118976:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075454.449528:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913119104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075454.449574:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913119232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075454.449610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913119360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075454.449645:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913119488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075454.449676:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913119616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075454.449752:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913119744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075454.641141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075455.153143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075455.153207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075455.665154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075456.177216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075456.689171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075457.201195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075457.201315:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075457.521492:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075457.521564:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075457.521593:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075457.521692:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075457.521706:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075457.521720:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075457.521736:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075457.521753:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075457.521768:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075457.521768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075454/real 1769075457] req@ffff935002a60000 x1855008913119488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075457.521814:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913119488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075457.521834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075457.521843:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075457.521850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913119488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075457.521863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075454/real 1769075457] req@ffff935002a62a00 x1855008913119104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075457.521883:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913119104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075457.521897:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075457.521903:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075457.521909:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913119104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075457.521920:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075454/real 1769075457] req@ffff935002a61880 x1855008913119232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075457.521939:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913119232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075457.521968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075457.521972:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075457.521976:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913119232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075457.521985:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075454/real 1769075457] req@ffff935002a62680 x1855008913119360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075457.522023:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913119360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075457.522034:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075457.522039:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075457.522043:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913119360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075457.522088:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075454/real 1769075457] req@ffff935002a61c00 x1855008913118976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075457.522106:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913118976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075457.522117:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075457.522122:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075457.522127:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913118976:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075457.522167:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075454/real 1769075457] req@ffff935002a62300 x1855008913119744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075457.522185:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913119744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075457.522196:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075457.522203:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075457.522208:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913119744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075457.522219:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075454/real 1769075457] req@ffff935002a61500 x1855008913119616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075457.522236:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913119616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075457.522247:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075457.522253:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075457.522257:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913119616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075457.714103:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075458.225178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075458.737177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075459.249189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075459.249361:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075459.569311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075459.569327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075459.569333:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075459.569349:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1165 00000100:00080000:1.0:1769075459.569364:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075459.569373:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075459.569425:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913119872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075459.569467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075459.569474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075459.569476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075459.569481:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1165 00000100:00080000:1.0:1769075459.569486:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075459.569490:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075459.569504:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075459.569522:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913120000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075459.569536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075459.569541:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075459.569544:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075459.569548:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1165 00000100:00080000:1.0:1769075459.569552:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075459.569556:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075459.569565:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075459.569568:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913119872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769075459.569582:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913120128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075459.569598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075459.569602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075459.569606:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075459.569613:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1165 00000100:00080000:1.0:1769075459.569620:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075459.569627:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075459.569654:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075459.569678:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913120256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075459.569700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769075459.569704:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769075459.569711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075459.569719:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075459.569731:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1165 00000100:00080000:1.0:1769075459.569741:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075459.569750:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075459.569762:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075459.569793:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075459/real 1769075459] req@ffff9350062a6680 x1855008913119872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075459.569796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913120384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075459.569867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075459.569876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075459.569880:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075459.569887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a6680 x1855008913119872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075459.569895:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1165 00000100:00080000:1.0:1769075459.569911:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075459.569927:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:3.0:1769075459.569932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769075459.569941:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769075459.569951:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075459.570000:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913119872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075459.570017:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769075459.570034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913120512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075459.570133:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769075459.570140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075459.570150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075459.570155:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075459.570166:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1165 00000100:00100000:3.0:1769075459.570177:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075459/real 1769075459] req@ffff9350062a4a80 x1855008913120000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075459.570180:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075459.570198:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075459.570224:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075459.570232:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913120000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075459.570259:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769075459.570261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913120640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075459.570278:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075459.570297:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075459.570364:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075459.570435:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075459.570461:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075459/real 1769075459] req@ffff9350062a5880 x1855008913120128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075459.570488:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913120128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075459.570506:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075459.570514:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075459.570521:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075459.570535:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075459.571158:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075459.571193:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075459/real 1769075459] req@ffff9350062a5180 x1855008913120256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075459.571229:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913120256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075459.571246:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075459.571253:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075459.571259:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075459.571271:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913120384:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075459.571345:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075459.571384:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075459/real 1769075459] req@ffff9350062a5c00 x1855008913120384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075459.571411:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913120384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075459.571441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075459.571447:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075459.571453:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913120384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075459.571466:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913120512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075459.571508:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075459.571533:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075459/real 1769075459] req@ffff9350062a5500 x1855008913120512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075459.571556:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913120512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075459.571585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075459.571591:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075459.571599:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913120512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075459.571612:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913120640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075459.571653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075459.571687:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075459/real 1769075459] req@ffff9350062a4380 x1855008913120640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075459.571713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913120640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075459.571731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075459.571737:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075459.571744:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913120640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075459.761212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075460.273191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075460.273436:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075460.785143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075461.297239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075461.297484:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075461.809246:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075462.321152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075462.321331:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075462.833140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075463.345181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075463.345336:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075463.857134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075464.369140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075464.369333:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075464.689505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913120768:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075464.689675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075464.689745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075464/real 1769075464] req@ffff935002a60a80 x1855008913120768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075464.689780:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913120768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075464.689832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075464.689856:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075464.689925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913120768:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075464.689993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120896:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075464.690027:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075464.690040:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075464/real 1769075464] req@ffff935002a61f80 x1855008913120896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075464.690086:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913120896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075464.690095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075464.690099:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075464.690102:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913120896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075464.690110:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075464.690128:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075464.690138:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075464/real 1769075464] req@ffff935002a63100 x1855008913121024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075464.690155:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913121024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075464.690162:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075464.690165:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075464.690168:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075464.690173:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075464.690187:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075464.690195:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075464/real 1769075464] req@ffff935002a61180 x1855008913121152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075464.690205:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913121152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075464.690212:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075464.690215:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075464.690217:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075464.690222:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913121280:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075464.690236:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075464.690243:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075464/real 1769075464] req@ffff935002a61500 x1855008913121280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075464.690253:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913121280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075464.690266:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075464.690268:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075464.690271:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913121280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075464.690275:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913121408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075464.690288:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075464.690296:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075464/real 1769075464] req@ffff935002a62300 x1855008913121408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075464.690305:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913121408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075464.690311:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075464.690314:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075464.690316:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913121408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075464.690321:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913121536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075464.690334:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075464.690341:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075464/real 1769075464] req@ffff935002a61c00 x1855008913121536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075464.690350:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913121536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075464.690357:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075464.690360:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075464.690363:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913121536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075464.881144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075465.393153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075465.393217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075465.905271:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075466.417178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075466.417294:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075466.929117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075467.441183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075467.441272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075467.953180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075468.465136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075468.465281:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075468.977153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075469.489176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075469.489253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075469.809469:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913121664:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075469.809540:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075469.809595:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075469/real 1769075469] req@ffff935002a62680 x1855008913121664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075469.809644:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913121664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075469.809660:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075469.809672:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075469.809679:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913121664:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075469.809689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121792:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075469.809718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075469.809735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075469/real 1769075469] req@ffff935002a61880 x1855008913121792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075469.809748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913121792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075469.809759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075469.809762:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075469.809766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075469.809828:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075469.809852:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075469.809864:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075469/real 1769075469] req@ffff935002a62a00 x1855008913121920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075469.809875:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913121920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075469.809883:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075469.809887:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075469.809983:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913121920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075469.809991:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075469.810014:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075469.810027:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075469/real 1769075469] req@ffff935002a60000 x1855008913122048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075469.810038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913122048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075469.810077:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075469.810082:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075469.810086:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075469.810093:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913122176:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075469.810115:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075469.810127:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075469/real 1769075469] req@ffff935002a61c00 x1855008913122176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075469.810138:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913122176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075469.810147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075469.810150:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075469.810153:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913122176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075469.810159:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913122304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075469.810177:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075469.810187:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075469/real 1769075469] req@ffff935002a62300 x1855008913122304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075469.810200:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913122304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075469.810207:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075469.810211:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075469.810214:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913122304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075469.810220:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913122432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075469.810240:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075469.810252:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075469/real 1769075469] req@ffff935002a61500 x1855008913122432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075469.810265:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913122432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075469.810273:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075469.810276:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075469.810286:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913122432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075470.001168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075470.513202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075470.513299:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075471.025145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075471.537144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075471.537262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075472.049121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075472.561189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075472.561264:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075473.073125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075473.585219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075473.585366:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075474.097134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075474.609222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075474.609345:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075474.929201:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075474.929214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075474.929220:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075474.929236:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1180 00000100:00080000:1.0:1769075474.929251:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075474.929262:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075474.929337:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913122560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075474.929385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075474.929390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075474.929393:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075474.929399:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1180 00000100:00080000:1.0:1769075474.929405:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075474.929411:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075474.929426:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075474.929468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913122688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075474.929468:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913122560:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075474.929490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075474.929493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075474.929495:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075474.929499:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1180 00000100:00080000:1.0:1769075474.929521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075474.929526:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075474.929535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075474.929553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913122816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075474.929566:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075474.929571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075474.929573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075474.929578:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1180 00000100:00080000:1.0:1769075474.929583:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075474.929590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075474.929595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1769075474.929605:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075474.929612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913122944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075474.929628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075474.929633:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075474.929636:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075474.929659:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1180 00000100:00080000:1.0:1769075474.929662:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075474.929666:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075474.929671:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075474.929688:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913123072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075474.929692:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9350062a5f80 x1855008913122560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075474.929739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075474.929747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075474.929749:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075474.929756:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1180 00000100:00080000:1.0:1769075474.929760:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075474.929762:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913122560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075474.929769:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075474.929782:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769075474.929789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.929799:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769075474.929801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913123200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075474.929814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075474.929816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913122560:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075474.929823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075474.929828:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075474.929852:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1180 00000100:00080000:1.0:1769075474.929857:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075474.929862:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075474.929868:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075474.929883:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769075474.929887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913123328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075474.929935:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075474.929957:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9350062a4380 x1855008913122688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.929974:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913122688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.929986:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.929990:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075474.929995:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075474.930003:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075474.930024:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075474.930035:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9350062a5500 x1855008913122816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.930049:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913122816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.930059:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.930099:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075474.930105:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075474.930113:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075474.930137:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075474.930152:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9350062a5c00 x1855008913122944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.930167:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913122944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.930178:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.930182:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075474.930189:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913122944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075474.930200:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913123072:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075474.930249:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075474.930266:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9350062a5180 x1855008913123072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.930286:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913123072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.930299:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.930304:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075474.930309:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913123072:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075474.930317:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913123200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075474.930361:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075474.930383:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9350062a5880 x1855008913123200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.930397:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913123200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.930409:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.930413:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075474.930417:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913123200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075474.930455:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913123328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075474.930472:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075474.930481:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9350062a4a80 x1855008913123328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.930493:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913123328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.930500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.930503:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075474.930505:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913123328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075475.121224:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075475.633161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075475.633338:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075476.145165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075476.657154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075476.657269:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075477.169233:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075477.681188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075477.681280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075478.193105:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075478.705198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075478.705353:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075479.217204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075479.729117:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075479.729201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075480.049138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075480.049149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075480.049155:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075480.049166:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1185 00000100:00080000:3.0:1769075480.049177:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075480.049183:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075480.049228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913123456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075480.049266:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075480.049270:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075480.049273:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075480.049278:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1185 00000100:00080000:3.0:1769075480.049282:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075480.049287:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075480.049300:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075480.049319:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913123584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075480.049335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075480.049340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075480.049342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075480.049347:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1185 00000100:00080000:3.0:1769075480.049352:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075480.049355:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075480.049361:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075480.049371:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913123712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075480.049383:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075480.049389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075480.049392:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075480.049414:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1185 00000100:00080000:3.0:1769075480.049420:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075480.049424:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075480.049430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075480.049443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913123840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075480.049452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075480.049459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075480.049463:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075480.049475:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1185 00000100:00080000:3.0:1769075480.049482:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075480.049488:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075480.049493:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075480.049509:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913123968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075480.049527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075480.049533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075480.049537:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075480.049545:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1185 00000100:00080000:3.0:1769075480.049552:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075480.049556:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075480.049560:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075480.049571:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913124096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075480.049591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075480.049595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075480.049597:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075480.049602:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1185 00000100:00080000:3.0:1769075480.049637:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075480.049659:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075480.049664:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075480.049680:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913124224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075480.241189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075480.753160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075480.753342:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075481.265245:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075481.777171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075481.777290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075482.289144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075482.801160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075482.801227:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075483.313170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075483.825144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075483.825192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075484.337109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075484.849176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075484.849238:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075485.169219:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075485.169232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075485.169237:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075485.169249:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1190 00000100:00080000:3.0:1769075485.169271:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075485.169277:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075485.169325:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913124352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075485.169350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075485.169353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075485.169354:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075485.169358:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1190 00000100:00080000:3.0:1769075485.169360:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075485.169363:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075485.169373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075485.169383:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913124480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075485.169401:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075485.169404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075485.169405:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075485.169410:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1190 00000100:00080000:3.0:1769075485.169413:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075485.169419:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075485.169423:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075485.169431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913124608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075485.169443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075485.169447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075485.169448:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075485.169451:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1190 00000100:00080000:3.0:1769075485.169453:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075485.169455:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075485.169458:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075485.169465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913124736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075485.169471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075485.169475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075485.169476:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075485.169479:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1190 00000100:00080000:3.0:1769075485.169482:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075485.169489:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075485.169493:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075485.169502:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913124864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075485.169509:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075485.169512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075485.169513:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075485.169515:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1190 00000100:00080000:3.0:1769075485.169519:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075485.169522:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075485.169526:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075485.169575:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913124992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075485.169583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075485.169586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075485.169590:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075485.169599:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1190 00000100:00080000:3.0:1769075485.169604:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075485.169609:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075485.169615:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075485.169628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913125120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075485.361179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075485.873132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075485.873223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075486.385130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075486.897160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075486.897279:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075487.409175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075487.921172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075487.921273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075488.433118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075488.945152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075488.945244:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075489.457175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075489.969180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075489.969253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075490.289155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075490.289166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075490.289171:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075490.289182:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1195 00000100:00080000:1.0:1769075490.289192:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075490.289198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075490.289242:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a6680 x1855008913125248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075490.289276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075490.289280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075490.289283:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075490.289288:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1195 00000100:00080000:1.0:1769075490.289291:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075490.289294:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075490.289304:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075490.289318:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913125376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075490.289327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075490.289332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075490.289334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075490.289342:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1195 00000100:00080000:1.0:1769075490.289345:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075490.289347:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075490.289351:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075490.289360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913125504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075490.289370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075490.289373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075490.289374:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075490.289377:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1195 00000100:00080000:1.0:1769075490.289379:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075490.289381:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075490.289384:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075490.289391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913125632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075490.289397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075490.289400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075490.289401:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075490.289403:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1195 00000100:00080000:1.0:1769075490.289407:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075490.289416:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075490.289423:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075490.289437:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913125760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075490.289464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075490.289468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075490.289470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075490.289475:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1195 00000100:00080000:1.0:1769075490.289480:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075490.289485:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075490.289491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075490.289506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913125888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075490.289513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075490.289518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075490.289521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075490.289529:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1195 00000100:00080000:1.0:1769075490.289534:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075490.289541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075490.289545:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075490.289554:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913126016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075490.481100:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075490.993142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075490.993231:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075491.505112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075492.017113:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075492.017225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075492.529106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075493.041142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075493.041259:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075493.553155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075494.065160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075494.065251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075494.577140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075495.089138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075495.089246:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075495.409375:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913126144:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075495.409453:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075495.409504:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075495/real 1769075495] req@ffff935002a61180 x1855008913126144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075495.409545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913126144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075495.409581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075495.409587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075495.409591:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913126144:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075495.409615:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913126272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075495.409635:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075495.409644:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075495/real 1769075495] req@ffff935002a63100 x1855008913126272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075495.409653:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913126272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075495.409660:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075495.409663:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075495.409665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913126272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075495.409669:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913126400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075495.409680:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075495.409686:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075495/real 1769075495] req@ffff935002a61f80 x1855008913126400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075495.409693:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913126400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075495.409699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075495.409700:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075495.409702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913126400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075495.409706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913126528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075495.409716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075495.409721:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075495/real 1769075495] req@ffff935002a60a80 x1855008913126528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075495.409729:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913126528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075495.409734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075495.409736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075495.409738:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913126528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075495.409743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913126656:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075495.409760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075495.409772:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075495/real 1769075495] req@ffff935002a61500 x1855008913126656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075495.409785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913126656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075495.409794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075495.409798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075495.409802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913126656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075495.409807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913126784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075495.409822:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075495.409829:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075495/real 1769075495] req@ffff935002a62300 x1855008913126784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075495.409837:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913126784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075495.409843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075495.409845:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075495.409847:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913126784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075495.409850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913126912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075495.409859:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075495.409865:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075495/real 1769075495] req@ffff935002a61c00 x1855008913126912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075495.409876:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913126912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075495.409881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075495.409883:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075495.409885:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913126912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075495.601177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075496.113154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075496.113247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075496.625218:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075497.137129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075497.137229:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075497.649229:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075498.161145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075498.161217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075498.673157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075499.185116:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075499.185239:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075499.697135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075500.209180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075500.209255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075500.529316:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913127040:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075500.529385:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075500.529428:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075500/real 1769075500] req@ffff935002a60000 x1855008913127040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075500.529466:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913127040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075500.529487:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075500.529496:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075500.529504:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913127040:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075500.529535:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913127168:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075500.529558:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075500.529570:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075500/real 1769075500] req@ffff935002a62a00 x1855008913127168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075500.529579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913127168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075500.529585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075500.529588:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075500.529590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913127168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075500.529594:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913127296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075500.529606:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075500.529611:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075500/real 1769075500] req@ffff935002a61880 x1855008913127296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075500.529619:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913127296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075500.529624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075500.529626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075500.529628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913127296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075500.529631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913127424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075500.529641:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075500.529646:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075500/real 1769075500] req@ffff935002a62680 x1855008913127424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075500.529654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913127424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075500.529659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075500.529660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075500.529662:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913127424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075500.529665:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913127552:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075500.529674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075500.529680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075500/real 1769075500] req@ffff935002a61c00 x1855008913127552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075500.529687:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913127552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075500.529693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075500.529694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075500.529696:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913127552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075500.529700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913127680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075500.529709:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075500.529714:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075500/real 1769075500] req@ffff935002a62300 x1855008913127680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075500.529721:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913127680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075500.529731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075500.529733:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075500.529735:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913127680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075500.529745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913127808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075500.529754:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075500.529759:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075500/real 1769075500] req@ffff935002a61500 x1855008913127808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075500.529767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913127808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075500.529772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075500.529774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075500.529776:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913127808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075500.721206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075501.233117:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075501.233204:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075501.745159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075502.257142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075502.257208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075502.769187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075503.281194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075503.281295:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075503.793130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075504.305174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075504.305336:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075504.817275:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075505.329180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075505.329287:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075505.649364:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913127936:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075505.649447:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075505.649509:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075505/real 1769075505] req@ffff935002a60a80 x1855008913127936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075505.649557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913127936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075505.649567:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075505.649572:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075505.649576:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913127936:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075505.649616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913128064:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075505.649637:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075505.649647:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075505/real 1769075505] req@ffff935002a61f80 x1855008913128064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075505.649657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913128064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075505.649663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075505.649666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075505.649668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913128064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075505.649673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913128192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075505.649693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075505.649700:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075505/real 1769075505] req@ffff935002a63100 x1855008913128192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075505.649714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913128192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075505.649720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075505.649722:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075505.649724:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913128192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075505.649728:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913128320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075505.649737:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075505.649743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075505/real 1769075505] req@ffff935002a61180 x1855008913128320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075505.649751:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913128320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075505.649756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075505.649758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075505.649760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913128320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075505.649763:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913128448:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075505.649773:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075505.649778:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075505/real 1769075505] req@ffff935002a61500 x1855008913128448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075505.649790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913128448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075505.649806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075505.649809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075505.649813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913128448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075505.649820:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913128576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075505.649842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075505.649854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075505/real 1769075505] req@ffff935002a62300 x1855008913128576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075505.649863:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913128576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075505.649869:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075505.649872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075505.649874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913128576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075505.649878:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913128704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075505.649892:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075505.649898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075505/real 1769075505] req@ffff935002a61c00 x1855008913128704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075505.649906:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913128704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075505.649911:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075505.649913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075505.649916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913128704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075505.841138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075506.353171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075506.353344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075506.865248:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075507.377178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075507.377336:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075507.889198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075508.401165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075508.401248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075508.913195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075509.425183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075509.425264:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075509.937111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075510.449186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075510.449263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075510.769255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075510.769272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075510.769279:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075510.769296:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1216 00000100:00080000:3.0:1769075510.769313:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075510.769321:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075510.769383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913128832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075510.769419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075510.769423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075510.769425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075510.769430:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1216 00000100:00080000:3.0:1769075510.769433:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075510.769438:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075510.769450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075510.769465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913128960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075510.769481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075510.769490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075510.769492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075510.769498:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1216 00000100:00080000:3.0:1769075510.769501:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075510.769505:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075510.769511:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075510.769522:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913129088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075510.769532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075510.769536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075510.769537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075510.769541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1216 00000100:00080000:3.0:1769075510.769554:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075510.769558:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075510.769569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075510.769586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913129216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075510.769603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075510.769609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075510.769613:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075510.769621:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1216 00000100:00080000:3.0:1769075510.769628:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075510.769642:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075510.769659:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075510.769690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913129344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075510.769725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075510.769730:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075510.769732:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075510.769738:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1216 00000100:00080000:3.0:1769075510.769754:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075510.769768:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075510.769778:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075510.769800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913129472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075510.769833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075510.769842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075510.769850:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075510.769868:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1216 00000100:00080000:3.0:1769075510.769878:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075510.769893:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075510.769912:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075510.769944:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913129600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075510.961203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075511.473137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075511.473263:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075511.985141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075512.497172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075512.497280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075513.009270:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075513.521209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075513.521254:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075514.033186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075514.545202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075514.545286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075515.057161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075515.569176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075515.569237:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075515.889194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075515.889205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075515.889209:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075515.889219:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1221 00000100:00080000:3.0:1769075515.889231:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075515.889237:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075515.889291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913129728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075515.889355:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075515.889360:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075515.889362:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075515.889366:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1221 00000100:00080000:3.0:1769075515.889370:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075515.889375:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075515.889386:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075515.889405:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913129856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075515.889423:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075515.889429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075515.889432:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075515.889457:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1221 00000100:00080000:3.0:1769075515.889462:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075515.889469:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075515.889483:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075515.889504:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913129984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075515.889520:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075515.889525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075515.889527:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075515.889549:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1221 00000100:00080000:3.0:1769075515.889562:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075515.889566:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075515.889572:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075515.889587:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913130112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075515.889595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075515.889599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075515.889600:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075515.889605:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1221 00000100:00080000:3.0:1769075515.889610:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075515.889614:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075515.889630:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075515.889646:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913130240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075515.889656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075515.889664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075515.889668:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075515.889677:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1221 00000100:00080000:3.0:1769075515.889681:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075515.889687:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075515.889692:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075515.889705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913130368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075515.889722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075515.889725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075515.889727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075515.889732:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1221 00000100:00080000:3.0:1769075515.889735:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075515.889743:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075515.889748:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075515.889760:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913130496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075516.081192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075516.593122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075516.593288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075517.105119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075517.617197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075517.617337:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075518.129155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075518.641146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075518.641276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075519.153153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075519.665193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075519.665335:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075520.177173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075520.689151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075520.689284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075521.009154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075521.009169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075521.009176:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075521.009189:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1226 00000100:00080000:3.0:1769075521.009206:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075521.009217:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075521.009271:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913130624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075521.009312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075521.009328:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075521.009331:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075521.009337:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1226 00000100:00080000:3.0:1769075521.009341:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075521.009348:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075521.009362:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075521.009384:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913130752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075521.009402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075521.009406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075521.009408:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075521.009414:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1226 00000100:00080000:3.0:1769075521.009424:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075521.009434:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075521.009454:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075521.009483:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913130880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075521.009500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075521.009508:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075521.009511:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075521.009519:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1226 00000100:00080000:3.0:1769075521.009547:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075521.009557:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075521.009570:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075521.009600:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913131008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075521.009621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075521.009627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075521.009631:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075521.009661:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1226 00000100:00080000:3.0:1769075521.009672:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075521.009683:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075521.009700:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075521.009757:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913131136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075521.009779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075521.009786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075521.009790:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075521.009802:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1226 00000100:00080000:3.0:1769075521.009815:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075521.009822:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075521.009830:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075521.009869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913131264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075521.009890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075521.009895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075521.009898:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075521.009908:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1226 00000100:00080000:3.0:1769075521.009913:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075521.009917:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075521.009924:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075521.009975:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913131392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075521.201164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075521.713179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075521.713293:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075522.225138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075522.737230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075522.737344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075523.249185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075523.761171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075523.761336:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075524.081441:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075524.081484:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075524.081503:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075524.081567:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075524.081589:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075524.081598:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075524.081606:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075524.081614:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075524.081623:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075524.273220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075524.785228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075524.785395:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075525.297133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075525.809176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075525.809315:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075526.129171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075526.129183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075526.129187:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075526.129199:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1231 00000100:00080000:3.0:1769075526.129212:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075526.129221:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075526.129266:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913131520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075526.129296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075526.129299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075526.129300:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075526.129304:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1231 00000100:00080000:3.0:1769075526.129307:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075526.129310:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075526.129321:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075526.129337:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913131648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075526.129346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075526.129350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075526.129353:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075526.129362:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1231 00000100:00080000:3.0:1769075526.129365:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075526.129367:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075526.129371:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075526.129380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913131776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075526.129388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075526.129391:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075526.129392:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075526.129394:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1231 00000100:00080000:3.0:1769075526.129396:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075526.129399:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075526.129402:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075526.129409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913131904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075526.129417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075526.129422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075526.129424:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075526.129440:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1231 00000100:00080000:3.0:1769075526.129447:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075526.129453:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075526.129460:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075526.129476:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913132032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075526.129489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075526.129498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075526.129503:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075526.129510:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1231 00000100:00080000:3.0:1769075526.129516:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075526.129526:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075526.129533:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075526.129552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913132160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075526.129563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075526.129567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075526.129569:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075526.129573:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1231 00000100:00080000:3.0:1769075526.129579:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075526.129587:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075526.129594:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075526.129609:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913132288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075526.321165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075526.833158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075526.833251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075527.345116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075527.857220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075527.857332:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075528.369305:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075528.881190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075528.881306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075529.393151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075529.905151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075529.905237:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075530.417145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075530.929204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075530.929281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075531.249133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075531.249147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075531.249152:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075531.249165:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1236 00000100:00080000:3.0:1769075531.249181:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075531.249192:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075531.249262:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913132416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075531.249303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075531.249309:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075531.249313:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075531.249318:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1236 00000100:00080000:3.0:1769075531.249323:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075531.249329:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075531.249342:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075531.249361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913132544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075531.249376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075531.249382:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075531.249394:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075531.249400:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1236 00000100:00080000:3.0:1769075531.249404:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075531.249407:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075531.249414:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075531.249426:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913132672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075531.249436:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075531.249440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075531.249442:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075531.249446:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1236 00000100:00080000:3.0:1769075531.249450:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075531.249457:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075531.249463:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075531.249475:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913132800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075531.249484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075531.249488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075531.249490:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075531.249495:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1236 00000100:00080000:3.0:1769075531.249500:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075531.249504:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075531.249511:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075531.249524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913132928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075531.249561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075531.249565:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075531.249567:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075531.249572:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1236 00000100:00080000:3.0:1769075531.249575:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075531.249582:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075531.249590:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075531.249606:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913133056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075531.249617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075531.249621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075531.249627:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075531.249639:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1236 00000100:00080000:3.0:1769075531.249645:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075531.249652:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075531.249656:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075531.249665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913133184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075531.441228:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075531.953204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075531.953402:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075532.465225:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075532.977192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075532.977290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075533.489190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075534.001142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075534.001238:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075534.513185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075535.025177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075535.025250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075535.537166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075536.049118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075536.049308:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075536.561159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075537.073132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075537.073280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075537.585162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075538.097164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075538.097263:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075538.609142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075539.121184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075539.121257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075539.633169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075540.145175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075540.145268:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075540.657211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075541.169140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075541.169207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075541.681257:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075542.193177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075542.193394:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075542.705174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075543.217191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075543.217357:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075543.729215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075544.241234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075544.241457:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075544.753225:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075545.265202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075545.265318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075545.777263:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075546.289234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075546.289362:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075546.801252:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075547.313284:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075547.313393:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075547.825222:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075548.337254:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075548.337417:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075548.849147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075549.361174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075549.361372:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075549.873269:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075550.385197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075550.385264:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075550.897177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075551.409162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075551.409309:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075551.729180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075551.729191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075551.729195:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075551.729207:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1257 00000100:00080000:1.0:1769075551.729219:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075551.729226:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075551.729268:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913136000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075551.729295:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075551.729298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075551.729299:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075551.729303:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1257 00000100:00080000:1.0:1769075551.729310:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075551.729313:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075551.729322:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075551.729334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913136128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075551.729343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075551.729347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075551.729349:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075551.729353:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1257 00000100:00080000:1.0:1769075551.729355:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075551.729358:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075551.729362:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075551.729370:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913136256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075551.729382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075551.729397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075551.729399:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075551.729404:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1257 00000100:00080000:1.0:1769075551.729408:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075551.729411:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075551.729432:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075551.729448:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913136384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075551.729457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075551.729460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075551.729461:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075551.729466:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1257 00000100:00080000:1.0:1769075551.729475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075551.729480:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075551.729487:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075551.729505:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913136512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075551.729523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075551.729528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075551.729531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075551.729538:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1257 00000100:00080000:1.0:1769075551.729545:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075551.729551:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075551.729562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075551.729586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913136640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075551.729635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075551.729646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075551.729650:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075551.729656:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1257 00000100:00080000:1.0:1769075551.729661:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075551.729673:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075551.729679:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075551.729700:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913136768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075551.921156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075552.433201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075552.433328:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075552.945212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075553.457157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075553.457276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075553.969225:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075554.481177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075554.481293:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075554.993214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075555.505181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075555.505467:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075556.017223:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075556.529273:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075556.529406:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075556.849208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075556.849221:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075556.849227:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075556.849236:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1262 00000100:00080000:3.0:1769075556.849250:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075556.849258:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075556.849302:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913136896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075556.849350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075556.849357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075556.849360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075556.849365:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1262 00000100:00080000:3.0:1769075556.849369:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075556.849373:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075556.849384:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075556.849399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913137024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075556.849411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075556.849415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075556.849418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075556.849421:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1262 00000100:00080000:3.0:1769075556.849424:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075556.849428:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075556.849432:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075556.849443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913137152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075556.849452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075556.849456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075556.849458:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075556.849467:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1262 00000100:00080000:3.0:1769075556.849471:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075556.849476:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075556.849485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075556.849505:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913137280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075556.849520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075556.849525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075556.849528:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075556.849536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1262 00000100:00080000:3.0:1769075556.849543:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075556.849549:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075556.849555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075556.849575:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913137408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075556.849588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075556.849593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075556.849595:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075556.849601:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1262 00000100:00080000:3.0:1769075556.849624:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075556.849649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075556.849661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075556.849682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913137536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075556.849697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075556.849702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075556.849705:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075556.849716:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1262 00000100:00080000:3.0:1769075556.849743:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075556.849765:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075556.849778:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075556.849801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913137664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075557.041173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075557.553219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075557.553423:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075558.065220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075558.577154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075558.577221:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075559.089116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075559.601165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075559.601330:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075560.113120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075560.625209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075560.625252:0:25069:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075561.137171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075561.649158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075561.649343:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075561.969151:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075561.969164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075561.969168:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075561.969179:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1267 00000100:00080000:3.0:1769075561.969191:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075561.969198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075561.969240:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913137792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075561.969286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075561.969290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075561.969292:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075561.969296:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1267 00000100:00080000:3.0:1769075561.969299:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075561.969303:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075561.969314:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075561.969328:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913137920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075561.969337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075561.969342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075561.969346:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075561.969355:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1267 00000100:00080000:3.0:1769075561.969358:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075561.969360:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075561.969365:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075561.969376:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913138048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075561.969383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075561.969386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075561.969388:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075561.969392:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1267 00000100:00080000:3.0:1769075561.969408:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075561.969411:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075561.969414:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075561.969427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913138176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075561.969436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075561.969440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075561.969442:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075561.969445:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1267 00000100:00080000:3.0:1769075561.969452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075561.969455:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075561.969458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075561.969468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913138304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075561.969475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075561.969478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075561.969480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075561.969487:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1267 00000100:00080000:3.0:1769075561.969493:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075561.969500:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075561.969507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075561.969520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913138432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075561.969556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075561.969567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075561.969573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075561.969585:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1267 00000100:00080000:3.0:1769075561.969591:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075561.969596:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075561.969603:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075561.969620:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913138560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075562.161128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075562.673161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075562.673298:0:25069:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 12 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769075562.673306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075562.673309:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316a200/0x410246667097e25d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97dc6 expref: -99 pid: 23705 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769075562.673341:0:25069:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x2a:0x0]: losing layout lock 00010000:00010000:1.0:1769075562.673389:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5200/0x410246667097e336 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf98058 expref: -99 pid: 24870 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769075562.673412:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316b200/0x410246667097e32f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf98043 expref: -99 pid: 24870 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769075562.673419:0:25069:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x2e:0x0]: losing layout lock 00010000:00010000:1.0:1769075562.673424:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff93500316aa00/0x410246667097e2f7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97f94 expref: -99 pid: 24869 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769075562.673434:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5c00/0x410246667097e328 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf98019 expref: -99 pid: 24869 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769075562.673438:0:25069:0:(vvp_object.c:114:vvp_conf_set()) [0x200000401:0x2e:0x0]: losing layout lock 00010000:00010000:1.0:1769075562.673441:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935002ea5e00/0x410246667097e2fe lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x48bf02eaaaf97fb0 expref: -99 pid: 24869 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769075562.673448:0:25069:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935012356000/0x410246667097e2f0 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2c:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x48bf02eaaaf97f71 expref: -99 pid: 24293 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1769075562.673525:0:25069:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff935002a7ce00 x1855008913032960/t4294973394(4294973394) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074982 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cmp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769075562.673538:0:25069:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9350067ee000: modify RPC slot 3 is allocated opc 35, max 7 00000100:00100000:1.0:1769075562.673543:0:25069:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9350062a6680 x1855008913138816/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:1.0:1769075562.673566:0:25069:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9350062a6680 x1855008913138816/t0(0) o35->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1769075562.673574:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075563.185154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075563.697186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075563.697187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075563.697294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075564.209182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075564.721189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075564.721189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075564.721237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075565.233226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075565.745209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075565.745209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075565.745321:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075566.257160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075566.769200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075566.769314:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075566.769356:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:1.0:1769075567.089309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075567.089320:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075567.089325:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075567.089337:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1272 00000100:00080000:1.0:1769075567.089349:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075567.089356:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075567.089400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913138944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075567.089441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075567.089445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075567.089446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075567.089450:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1272 00000100:00080000:1.0:1769075567.089453:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075567.089456:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075567.089467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075567.089479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913139072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075567.089488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075567.089491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075567.089492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075567.089496:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1272 00000100:00080000:1.0:1769075567.089500:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075567.089503:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075567.089507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075567.089515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913139200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075567.089523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075567.089558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075567.089563:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075567.089568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1272 00000100:00080000:1.0:1769075567.089572:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075567.089576:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075567.089582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075567.089600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913139328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075567.089616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075567.089623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075567.089626:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075567.089632:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1272 00000100:00080000:1.0:1769075567.089639:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075567.089646:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075567.089658:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075567.089696:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913139456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075567.089714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075567.089717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075567.089719:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075567.089723:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1272 00000100:00080000:1.0:1769075567.089728:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075567.089753:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075567.089770:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075567.089797:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913139584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075567.089833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075567.089843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075567.089851:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075567.089865:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1272 00000100:00080000:1.0:1769075567.089878:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075567.089906:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075567.089936:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075567.089967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913139712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075567.281279:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075567.793275:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075567.793345:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075568.305221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075568.817171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075568.817198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075569.329231:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075569.841254:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075569.841261:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075570.353235:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075570.865208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075570.865223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075571.377178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075571.889215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075571.889222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075572.401219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075572.913186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075572.913257:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075572.913297:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075573.425169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075573.937128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075573.937237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075574.449181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075574.961193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075574.961209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075575.473206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075575.985184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075575.985190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075576.497109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075577.009137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075577.009138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075577.329369:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913140736:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075577.329480:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075577.329545:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075577/real 1769075577] req@ffff935002a62680 x1855008913140736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075577.329588:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913140736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075577.329629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075577.329640:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075577.329652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913140736:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075577.329694:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913140864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075577.329746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075577.329765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075577/real 1769075577] req@ffff935002a61880 x1855008913140864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075577.329782:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913140864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075577.329811:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075577.329816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075577.329821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913140864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075577.329829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913140992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075577.329856:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075577.329870:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075577/real 1769075577] req@ffff935002a62a00 x1855008913140992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075577.329886:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913140992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075577.329895:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075577.329899:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075577.329903:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913140992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075577.329910:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913141120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075577.329936:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075577.329949:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075577/real 1769075577] req@ffff935002a60000 x1855008913141120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075577.329963:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913141120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075577.329973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075577.329977:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075577.329983:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913141120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075577.330013:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913141248:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075577.330040:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075577.330114:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075577/real 1769075577] req@ffff935002a61c00 x1855008913141248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075577.330130:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913141248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075577.330140:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075577.330144:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075577.330148:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913141248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075577.330155:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913141376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075577.330681:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075577.330699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075577/real 1769075577] req@ffff935002a62300 x1855008913141376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075577.330714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913141376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075577.330724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075577.330728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075577.330731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913141376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075577.330759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913141504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075577.330781:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075577.330796:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075577/real 1769075577] req@ffff935002a61500 x1855008913141504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075577.330813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913141504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075577.330823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075577.330827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075577.330831:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913141504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075578.033136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075578.033169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075579.057107:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075579.057126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075580.081150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075580.081217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075580.081225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075581.105166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075581.105185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075581.105269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075582.129167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075582.129167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075582.449335:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913141632:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075582.449419:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075582.449467:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075582/real 1769075582] req@ffff935002a61180 x1855008913141632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075582.449507:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913141632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075582.449548:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075582.449559:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075582.449567:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913141632:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075582.449618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913141760:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075582.449671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075582.449693:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075582/real 1769075582] req@ffff935002a63100 x1855008913141760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075582.449711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913141760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075582.449722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075582.449726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075582.449730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913141760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075582.449737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913141888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075582.449763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075582.449777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075582/real 1769075582] req@ffff935002a61f80 x1855008913141888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075582.449813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913141888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075582.449823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075582.449828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075582.449832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913141888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075582.449840:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075582.449866:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075582.449881:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075582/real 1769075582] req@ffff935002a60a80 x1855008913142016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075582.449916:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913142016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075582.449924:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075582.449927:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075582.449930:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075582.449935:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913142144:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075582.449954:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075582.449963:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075582/real 1769075582] req@ffff935002a61500 x1855008913142144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075582.449973:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913142144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075582.449980:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075582.449983:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075582.449986:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913142144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075582.449990:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913142272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075582.450003:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075582.450010:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075582/real 1769075582] req@ffff935002a62300 x1855008913142272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075582.450019:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913142272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075582.450031:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075582.450034:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075582.450037:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913142272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075582.450079:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913142400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075582.450094:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075582.450104:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075582/real 1769075582] req@ffff935002a61c00 x1855008913142400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075582.450113:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913142400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075582.450120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075582.450123:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075582.450126:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913142400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075583.153199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075583.153199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075584.177157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075584.177164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075585.201108:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075585.201144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075585.201288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075586.225134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075586.225134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075587.249184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075587.249184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075587.569338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913142528:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075587.569451:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075587.569492:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075587.569513:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075587.569530:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075587.569543:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075587.569557:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075588.273195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075588.273202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075589.297178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075589.297178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075590.321213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075590.321215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000400:00000100:1.0:1769075590.641395:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075590.641455:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075590.641476:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075590.641524:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075590.641535:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075590.641544:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075590.641559:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075590.641570:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075590.641582:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075590.641593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075587/real 1769075590] req@ffff9350103fb480 x1855008913142784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075590.641627:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913142784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075590.641641:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075590.641647:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075590.641652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075590.641663:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075587/real 1769075590] req@ffff9350103f8380 x1855008913142912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075590.641678:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913142912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075590.641686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075590.641690:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075590.641694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075590.641702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075587/real 1769075590] req@ffff9350103faa00 x1855008913143040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075590.641717:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913143040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075590.641726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075590.641730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075590.641733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075590.641741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075587/real 1769075590] req@ffff9350103f9f80 x1855008913143168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075590.641755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913143168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075590.641792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075590.641796:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075590.641800:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075590.641809:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075587/real 1769075590] req@ffff9350103fbb80 x1855008913143296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075590.641823:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913143296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075590.641830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075590.641833:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075590.641836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075590.641847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075587/real 1769075590] req@ffff9350103fb100 x1855008913142528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075590.641862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913142528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075590.641871:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075590.641875:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075590.641879:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913142528:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075590.641887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075587/real 1769075590] req@ffff9350103f9180 x1855008913142656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075590.641901:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913142656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075590.641910:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075590.641914:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075590.641918:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913142656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075591.345182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075591.345190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075592.369138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075592.369138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:1.0:1769075592.689233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075592.689245:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075592.689249:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075592.689260:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1298 00000100:00080000:1.0:1769075592.689271:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075592.689277:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075592.689326:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913143424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075592.689354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075592.689358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075592.689361:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075592.689366:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1298 00000100:00080000:1.0:1769075592.689370:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075592.689374:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075592.689388:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075592.689409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913143552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075592.689422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075592.689428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075592.689430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075592.689430:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913143424:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075592.689441:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1298 00000100:00080000:1.0:1769075592.689446:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075592.689451:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075592.689457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075592.689481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913143680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075592.689493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075592.689496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075592.689498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075592.689502:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1298 00000100:00080000:1.0:1769075592.689507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075592.689511:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075592.689518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1769075592.689530:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075592.689535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913143808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075592.689550:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075592.689555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075592.689558:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075592.689562:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1298 00000100:00080000:1.0:1769075592.689567:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075592.689573:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075592.689580:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075592.689598:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075592/real 1769075592] req@ffff9350062a5f80 x1855008913143424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075592.689615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913143936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075592.689626:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913143424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075592.689636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075592.689648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075592.689659:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075592.689670:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1298 00000100:00080000:1.0:1769075592.689675:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075592.689676:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075592.689680:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:3.0:1769075592.689687:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769075592.689688:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075592.689697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913143424:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769075592.689710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913144064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075592.689724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075592.689744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075592.689747:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075592.689752:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1298 00000100:00080000:1.0:1769075592.689758:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075592.689760:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913143552:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769075592.689767:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075592.689776:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075592.689794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913144192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075592.689803:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075592.689830:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075592/real 1769075592] req@ffff9350062a4380 x1855008913143552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075592.689848:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913143552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075592.689859:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075592.689863:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075592.689867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913143552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075592.689874:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913143680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075592.689898:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075592.689911:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075592/real 1769075592] req@ffff9350062a5500 x1855008913143680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075592.689924:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913143680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075592.689934:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075592.689938:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075592.689942:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913143680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075592.689950:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913143808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075592.690642:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075592.690667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075592/real 1769075592] req@ffff9350062a5c00 x1855008913143808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075592.690682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913143808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075592.690693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075592.690697:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075592.690701:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913143808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075592.690708:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075592.690752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075592.690766:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075592/real 1769075592] req@ffff9350062a5180 x1855008913143936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075592.690778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913143936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075592.690789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075592.690793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075592.690797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913143936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075592.690805:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075592.690827:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075592.690839:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075592/real 1769075592] req@ffff9350062a5880 x1855008913144064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075592.690852:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913144064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075592.690862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075592.690865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075592.690868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075592.690897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075592.690919:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075592.690932:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075592/real 1769075592] req@ffff9350062a4a80 x1855008913144192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075592.690945:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913144192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075592.690963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075592.690966:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075592.690970:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075593.393226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075593.393239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075594.417151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075594.417151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075595.441155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075595.441183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075596.465158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075596.465158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075597.489171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075597.489217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075597.489320:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075597.809154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075597.809164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075597.809169:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075597.809177:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1303 00000100:00080000:1.0:1769075597.809188:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075597.809195:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075597.809235:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913144320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075597.809262:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075597.809264:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075597.809266:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075597.809269:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1303 00000100:00080000:1.0:1769075597.809271:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075597.809274:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075597.809283:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075597.809292:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913144448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075597.809300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075597.809303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075597.809306:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075597.809308:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913144320:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075597.809311:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1303 00000100:00080000:1.0:1769075597.809316:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075597.809319:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075597.809323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075597.809331:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913144576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075597.809337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075597.809340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075597.809341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075597.809344:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1303 00000100:00080000:1.0:1769075597.809347:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075597.809350:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075597.809353:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075597.809360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913144704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075597.809366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075597.809368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075597.809370:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075597.809373:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1303 00000100:00080000:1.0:1769075597.809376:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075597.809378:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075597.809381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075597.809384:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075597.809392:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913144832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075597.809400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075597.809403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075597.809405:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075597.809408:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1303 00000100:00080000:1.0:1769075597.809410:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075597.809412:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075597.809416:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075597.809424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913144960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075597.809430:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075597/real 1769075597] req@ffff9350062a4a80 x1855008913144320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075597.809445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075597.809454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075597.809457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075597.809461:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1303 00000100:00080000:1.0:1769075597.809464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075597.809468:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913144320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075597.809469:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075597.809475:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769075597.809486:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769075597.809490:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913145088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075597.809493:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075597.809501:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913144320:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075597.809534:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913144448:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075597.809561:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075597.809574:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075597/real 1769075597] req@ffff9350062a5880 x1855008913144448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075597.809584:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913144448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075597.809591:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075597.809593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075597.809596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913144448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075597.809600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913144576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075597.809614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075597.809620:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075597/real 1769075597] req@ffff9350062a5180 x1855008913144576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075597.809629:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913144576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075597.809639:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075597.809666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075597.809669:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913144576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075597.809673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913144704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075597.809688:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075597.809703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075597/real 1769075597] req@ffff9350062a5c00 x1855008913144704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075597.809711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913144704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075597.809716:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075597.809719:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075597.809721:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913144704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075597.809724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075597.809734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075597.809740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075597/real 1769075597] req@ffff9350062a5500 x1855008913144832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075597.809746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913144832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075597.809752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075597.809754:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075597.809756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075597.809759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075597.809769:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075597.809774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075597/real 1769075597] req@ffff9350062a4380 x1855008913144960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075597.809781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913144960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075597.809791:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075597.809793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075597.809795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913144960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075597.809798:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075597.809808:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075597.809813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075597/real 1769075597] req@ffff9350062a5f80 x1855008913145088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075597.809821:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913145088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075597.809827:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075597.809828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075597.809830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075598.513143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075598.513144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075598.513181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075599.537212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075599.537231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075599.537320:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075600.561117:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075600.561155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075600.561225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075601.585207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075601.585220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075601.585288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075602.609155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075602.609172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075602.609233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075602.929627:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913145216:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075602.929781:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075602.929861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9350103fb800 x1855008913145216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075602.929951:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913145216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075602.930019:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075602.930036:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075602.930093:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913145216:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075602.930109:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913145344:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075602.930164:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075602.930191:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9350103f9c00 x1855008913145344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075602.930208:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913145344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075602.930221:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075602.930226:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075602.930230:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913145344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075602.930243:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913145472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075602.930264:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075602.930275:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9350103f8700 x1855008913145472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075602.930290:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913145472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075602.930300:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075602.930303:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075602.930307:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913145472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075602.930346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913145600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075602.930366:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075602.930473:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9350103f9500 x1855008913145600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075602.930514:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913145600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075602.930529:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075602.930536:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075602.930542:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913145600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075602.930584:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075602.930665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075602.930692:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9350103fad80 x1855008913145728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075602.930711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913145728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075602.930722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075602.930726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075602.930731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075602.930739:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075602.930762:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075602.930773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9350103f9180 x1855008913145856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075602.930788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913145856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075602.930798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075602.930801:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075602.930805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075602.930812:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075602.930829:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075602.930840:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9350103fb100 x1855008913145984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075602.930854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913145984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075602.930873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075602.930877:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075602.930883:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913145984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075603.633139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075603.633145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075603.633209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075604.657194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075604.657194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075604.657231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075605.681150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075605.681217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075606.705171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075606.705269:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075606.705378:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075607.729134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075607.729158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075607.729163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075608.049195:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075608.049205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075608.049210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075608.049221:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1313 00000100:00080000:3.0:1769075608.049233:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075608.049241:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075608.049291:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913146112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075608.049324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075608.049328:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075608.049330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075608.049333:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1313 00000100:00080000:3.0:1769075608.049337:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075608.049340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075608.049350:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075608.049361:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913146240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075608.049375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075608.049377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075608.049379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075608.049385:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1313 00000100:00080000:3.0:1769075608.049389:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075608.049395:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075608.049399:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075608.049407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913146368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075608.049414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075608.049417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075608.049418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075608.049421:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1313 00000100:00080000:3.0:1769075608.049423:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075608.049425:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075608.049429:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075608.049435:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913146496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075608.049441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075608.049443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075608.049444:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075608.049446:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1313 00000100:00080000:3.0:1769075608.049457:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075608.049460:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075608.049463:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075608.049502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913146624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075608.049510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075608.049513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075608.049514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075608.049517:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1313 00000100:00080000:3.0:1769075608.049521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075608.049523:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075608.049527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075608.049540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913146752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075608.049561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075608.049569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075608.049571:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075608.049574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1313 00000100:00080000:3.0:1769075608.049576:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075608.049579:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075608.049582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075608.049591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913146880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075608.753179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075608.753179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075608.753215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075609.777142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075609.777144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075609.777230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075610.801240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075610.801241:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075611.825177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075611.825180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075612.849199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075612.849199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075613.169356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913147008:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075613.169444:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075613.169492:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075613/real 1769075613] req@ffff935002a60000 x1855008913147008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075613.169520:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913147008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075613.169533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075613.169543:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075613.169551:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913147008:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075613.169583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913147136:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075613.169603:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075613.169613:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075613/real 1769075613] req@ffff935002a62a00 x1855008913147136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075613.169627:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913147136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075613.169637:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075613.169641:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075613.169646:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913147136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075613.169653:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913147264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075613.169675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075613.169694:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075613/real 1769075613] req@ffff935002a61880 x1855008913147264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075613.169702:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913147264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075613.169708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075613.169710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075613.169712:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913147264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075613.169716:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913147392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075613.169729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075613.169735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075613/real 1769075613] req@ffff935002a62680 x1855008913147392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075613.169743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913147392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075613.169748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075613.169750:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075613.169751:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913147392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075613.169755:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913147520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075613.169764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075613.169770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075613/real 1769075613] req@ffff935002a61c00 x1855008913147520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075613.169778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913147520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075613.169783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075613.169785:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075613.169787:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913147520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075613.169790:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913147648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075613.169799:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075613.169804:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075613/real 1769075613] req@ffff935002a62300 x1855008913147648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075613.169810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913147648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075613.169815:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075613.169817:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075613.169819:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913147648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075613.169838:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913147776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075613.169849:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075613.169855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075613/real 1769075613] req@ffff935002a61500 x1855008913147776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075613.169862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913147776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075613.169868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075613.169870:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075613.169872:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913147776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075613.873136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075613.873164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075613.873211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075614.897231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075614.897241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075614.897284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075615.921133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075615.921166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075616.945147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075616.945148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075617.969140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075617.969162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075618.289338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913147904:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075618.289416:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075618.289494:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075618/real 1769075618] req@ffff935002a60a80 x1855008913147904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075618.289525:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913147904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075618.289558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075618.289567:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075618.289573:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913147904:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075618.289625:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075618.289652:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075618.289685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075618/real 1769075618] req@ffff935002a61f80 x1855008913148032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075618.289723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913148032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075618.289759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075618.289767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075618.289773:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075618.289784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075618.289810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075618.289840:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075618/real 1769075618] req@ffff935002a63100 x1855008913148160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075618.289850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913148160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075618.289855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075618.289858:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075618.289861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075618.289868:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075618.289904:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075618.289915:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075618/real 1769075618] req@ffff935002a61180 x1855008913148288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075618.289928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913148288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075618.289943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075618.289945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075618.289948:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075618.289960:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913148416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075618.289975:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075618.289983:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075618/real 1769075618] req@ffff935002a61500 x1855008913148416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075618.289991:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913148416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075618.289997:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075618.289999:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075618.290001:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913148416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075618.290004:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913148544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075618.290013:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075618.290019:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075618/real 1769075618] req@ffff935002a62300 x1855008913148544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075618.290026:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913148544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075618.290031:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075618.290032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075618.290035:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913148544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075618.290039:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913148672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075618.290047:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075618.290053:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075618/real 1769075618] req@ffff935002a61c00 x1855008913148672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075618.290059:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913148672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075618.290065:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075618.290066:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075618.290068:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913148672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075618.993208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075618.993212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075620.017180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075620.017181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075621.041142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075621.041142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075622.065110:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075622.065131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075623.089122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075623.089122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075623.409350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913148800:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075623.409421:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075623.409468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075623/real 1769075623] req@ffff9350103fbb80 x1855008913148800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075623.409497:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913148800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075623.409513:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075623.409518:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075623.409523:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913148800:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075623.409552:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148928:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075623.409576:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075623.409594:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075623/real 1769075623] req@ffff9350103f9f80 x1855008913148928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075623.409618:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913148928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075623.409631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075623.409636:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075623.409641:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913148928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075623.409648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075623.409671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075623.409680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075623/real 1769075623] req@ffff9350103faa00 x1855008913149056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075623.409689:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913149056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075623.409697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075623.409701:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075623.409705:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075623.409712:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075623.409733:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075623.409745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075623/real 1769075623] req@ffff9350103f8380 x1855008913149184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075623.409756:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913149184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075623.409762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075623.409764:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075623.409766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075623.409770:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913149312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075623.409784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075623.409791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075623/real 1769075623] req@ffff9350103fb480 x1855008913149312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075623.409807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913149312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075623.409813:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075623.409815:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075623.409817:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913149312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075623.409831:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913149440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075623.409842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075623.409849:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075623/real 1769075623] req@ffff9350103fbb80 x1855008913149440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075623.409856:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913149440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075623.409862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075623.409864:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075623.409866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913149440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075623.409869:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913149568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075623.409878:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075623.409883:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075623/real 1769075623] req@ffff9350103fb100 x1855008913149568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075623.409890:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913149568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075623.409895:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075623.409897:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075623.409899:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913149568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075624.113138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075624.113138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075625.137142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075625.137142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075625.137221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075625.137234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075626.161160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075626.161160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075627.185112:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075627.185177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075628.209223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075628.209259:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:1.0:1769075628.529186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075628.529202:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075628.529207:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075628.529219:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1334 00000100:00080000:1.0:1769075628.529232:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075628.529241:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075628.529288:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913149696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075628.529318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075628.529321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075628.529323:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075628.529334:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1334 00000100:00080000:1.0:1769075628.529338:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075628.529345:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075628.529355:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075628.529366:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913149696:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769075628.529367:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913149824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075628.529381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075628.529384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075628.529385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075628.529388:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1334 00000100:00080000:1.0:1769075628.529392:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075628.529394:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075628.529398:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075628.529407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913149952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075628.529415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075628.529417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075628.529418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075628.529420:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1334 00000100:00080000:1.0:1769075628.529423:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075628.529427:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075628.529430:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1769075628.529438:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075628.529441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913150080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075628.529449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075628.529452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075628.529454:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075628.529457:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1334 00000100:00080000:1.0:1769075628.529460:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075628.529463:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075628.529466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075628.529474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913150208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075628.529481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075628.529485:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075628/real 1769075628] req@ffff9350062a5f80 x1855008913149696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075628.529486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075628.529489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075628.529496:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1334 00000100:00080000:1.0:1769075628.529502:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075628.529507:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075628.529514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075628.529525:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913149696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075628.529526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913150336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075628.529559:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075628.529561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075628.529565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075628.529566:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769075628.529567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075628.529574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1334 00000100:00100000:3.0:1769075628.529574:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913149696:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075628.529580:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075628.529584:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075628.529587:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075628.529594:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913150464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075628.529603:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075628.529622:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075628.529632:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075628/real 1769075628] req@ffff9350062a4380 x1855008913149824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075628.529641:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913149824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075628.529648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075628.529651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075628.529653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075628.529657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075628.529667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075628.529673:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075628/real 1769075628] req@ffff9350062a5500 x1855008913149952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075628.529680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913149952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075628.529690:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075628.529692:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075628.529694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913149952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075628.529697:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075628.529706:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075628.529712:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075628/real 1769075628] req@ffff9350062a5c00 x1855008913150080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075628.529719:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913150080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075628.529724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075628.529726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075628.529728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075628.529731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913150208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075628.529740:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075628.529746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075628/real 1769075628] req@ffff9350062a5180 x1855008913150208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075628.529753:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913150208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075628.529758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075628.529759:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075628.529762:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913150208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075628.529766:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913150336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075628.529775:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075628.529780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075628/real 1769075628] req@ffff9350062a5880 x1855008913150336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075628.529787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913150336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075628.529792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075628.529794:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075628.529795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913150336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075628.529807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913150464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075628.529816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075628.529822:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075628/real 1769075628] req@ffff9350062a4a80 x1855008913150464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075628.529828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913150464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075628.529833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075628.529834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075628.529836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913150464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075629.233194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075629.233203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075630.257187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075630.257189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075631.281219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075631.281255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075632.305183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075632.305197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075632.305252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075633.329146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075633.329147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075633.649539:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913150592:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075633.649607:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075633.649654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075633/real 1769075633] req@ffff9350103f9180 x1855008913150592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075633.649675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913150592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075633.649703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075633.649712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075633.649717:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913150592:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075633.649747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150720:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075633.649767:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075633.649777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075633/real 1769075633] req@ffff9350103fad80 x1855008913150720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075633.649786:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913150720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075633.649793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075633.649796:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075633.649798:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075633.649802:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075633.649814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075633.649820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075633/real 1769075633] req@ffff9350103f9500 x1855008913150848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075633.649827:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913150848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075633.649832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075633.649834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075633.649836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075633.649840:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075633.649850:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075633.649855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075633/real 1769075633] req@ffff9350103f8700 x1855008913150976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075633.649862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913150976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075633.649867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075633.649869:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075633.649871:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913150976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075633.649874:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913151104:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075633.649888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075633.649899:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075633/real 1769075633] req@ffff9350103f9c00 x1855008913151104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075633.649920:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913151104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075633.649929:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075633.649932:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075633.649936:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913151104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075633.649943:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913151232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075633.649964:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075633.649973:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075633/real 1769075633] req@ffff9350103fb800 x1855008913151232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075633.649982:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913151232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075633.649987:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075633.649989:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075633.649992:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913151232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075633.650007:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913151360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075633.650019:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075633.650025:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075633/real 1769075633] req@ffff9350103fb100 x1855008913151360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075633.650032:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913151360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075633.650141:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075633.650148:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075633.650156:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913151360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075634.353167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075634.353168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075635.377144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075635.377144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075636.401156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075636.401157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075637.425145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075637.425145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075637.425194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075638.449166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075638.449180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075638.449216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075638.769373:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913151488:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075638.769455:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075638.769506:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075638/real 1769075638] req@ffff9350103fbb80 x1855008913151488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075638.769540:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913151488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075638.769614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075638.769623:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075638.769637:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913151488:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075638.769682:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913151616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075638.769723:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075638.769751:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075638/real 1769075638] req@ffff9350103fb480 x1855008913151616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075638.769821:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913151616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075638.769831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075638.769834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075638.769836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913151616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075638.769843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913151744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075638.769862:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075638.769869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075638/real 1769075638] req@ffff9350103f8380 x1855008913151744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075638.769877:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913151744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075638.769882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075638.769884:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075638.769887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913151744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075638.769890:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913151872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075638.769900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075638.769905:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075638/real 1769075638] req@ffff9350103faa00 x1855008913151872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075638.769912:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913151872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075638.769922:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075638.769925:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075638.769927:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913151872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075638.769931:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152000:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075638.769940:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075638.769945:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075638/real 1769075638] req@ffff9350103f9f80 x1855008913152000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075638.769952:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913152000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075638.769983:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075638.769985:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075638.769988:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075638.770008:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075638.770023:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075638.770031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075638/real 1769075638] req@ffff9350103fbb80 x1855008913152128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075638.770039:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913152128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075638.770045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075638.770047:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075638.770090:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075638.770095:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075638.770109:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075638.770116:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075638/real 1769075638] req@ffff9350103fb100 x1855008913152256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075638.770124:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913152256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075638.770130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075638.770132:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075638.770134:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075639.473170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075639.473178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075640.497134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075640.497134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075641.521144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075641.521161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075641.521199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075642.545124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075642.545137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075642.545175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075643.569130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075643.569133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075643.889326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913152384:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075643.889402:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075643.889442:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff9350103fb800 x1855008913152384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.889464:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913152384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075643.889475:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075643.889484:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075643.889491:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913152384:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075643.889517:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913152512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075643.889547:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075643.889563:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff9350103f9c00 x1855008913152512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.889602:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913152512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075643.889612:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075643.889620:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075643.889626:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913152512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075643.889632:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913152640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075643.889649:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075643.889657:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff9350103f8700 x1855008913152640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.889665:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913152640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075643.889671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075643.889673:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075643.889675:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913152640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075643.889678:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913152768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075643.889689:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075643.889695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff9350103f9500 x1855008913152768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.889708:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913152768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075643.889717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075643.889721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075643.889724:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913152768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075643.889731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152896:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075643.889752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075643.889764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff9350103fad80 x1855008913152896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.889778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913152896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075643.889787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075643.889790:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075643.889792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913152896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075643.889809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075643.889825:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075643.889833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff9350103f9180 x1855008913153024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.889855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913153024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075643.889862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075643.889864:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075643.889888:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075643.889894:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075643.889909:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075643.889916:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff9350103fb800 x1855008913153152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.889936:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913153152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075643.889942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075643.889944:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075643.889946:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075644.593150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075644.593175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075645.617189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075645.617198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075645.617248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075646.641109:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075646.641142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075646.641208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075647.665155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075647.665155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075647.665210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075648.177161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075648.689137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075648.689206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075648.689291:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075649.009330:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913153280:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075649.009446:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075649.009506:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075649/real 1769075649] req@ffff9350103fb100 x1855008913153280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075649.009526:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913153280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075649.009535:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075649.009543:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075649.009547:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913153280:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075649.009554:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913153408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075649.009573:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075649.009582:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075649/real 1769075649] req@ffff9350103fbb80 x1855008913153408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075649.009591:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913153408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075649.009598:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075649.009601:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075649.009603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913153408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075649.009631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913153536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075649.009653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075649.009664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075649/real 1769075649] req@ffff9350103f9f80 x1855008913153536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075649.009679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913153536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075649.009685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075649.009688:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075649.009690:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913153536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075649.009695:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913153664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075649.009710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075649.009717:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075649/real 1769075649] req@ffff9350103faa00 x1855008913153664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075649.009725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913153664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075649.009730:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075649.009732:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075649.009734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913153664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075649.009738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153792:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075649.009748:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075649.009754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075649/real 1769075649] req@ffff9350103f8380 x1855008913153792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075649.009761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913153792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075649.009771:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075649.009774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075649.009776:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075649.009780:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075649.009809:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075649.009819:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075649/real 1769075649] req@ffff9350103fb480 x1855008913153920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075649.009831:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913153920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075649.009839:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075649.009842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075649.009846:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913153920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075649.009863:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913154048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075649.009885:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075649.009898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075649/real 1769075649] req@ffff9350103fb800 x1855008913154048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075649.009909:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913154048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075649.009917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075649.009920:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075649.009923:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913154048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075649.201180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075649.713208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075649.713216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075650.225136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075650.737136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075650.737136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075651.249140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075651.761142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075651.761142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075652.273225:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075652.785152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075652.785186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075653.297103:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075653.809179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075653.809212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075653.809258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075654.129158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075654.129170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075654.129180:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075654.129192:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1359 00000100:00080000:3.0:1769075654.129204:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075654.129211:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075654.129253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913154176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075654.129280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075654.129284:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075654.129285:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075654.129289:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1359 00000100:00080000:3.0:1769075654.129291:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075654.129294:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075654.129306:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075654.129317:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913154304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075654.129326:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075654.129328:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075654.129332:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075654.129342:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1359 00000100:00080000:3.0:1769075654.129351:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075654.129359:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075654.129367:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075654.129383:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913154432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075654.129395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075654.129399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075654.129401:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075654.129406:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1359 00000100:00080000:3.0:1769075654.129422:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075654.129428:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075654.129435:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075654.129449:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913154560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075654.129456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075654.129460:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075654.129464:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075654.129472:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1359 00000100:00080000:3.0:1769075654.129484:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075654.129492:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075654.129499:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075654.129533:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913154688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075654.129546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075654.129549:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075654.129551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075654.129556:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1359 00000100:00080000:3.0:1769075654.129562:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075654.129566:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075654.129572:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075654.129591:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913154816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075654.129598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075654.129601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075654.129603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075654.129606:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1359 00000100:00080000:3.0:1769075654.129609:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075654.129613:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075654.129617:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075654.129628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913154944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075654.321156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075654.833157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075654.833200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075654.833233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075655.345135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075655.857186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075655.857228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075655.857308:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075656.369163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075656.881150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075656.881223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075657.201384:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075657.201432:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075657.201454:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075657.201496:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075657.201508:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075657.201518:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075657.201530:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075657.201539:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075657.201550:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075657.393131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075657.905250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075657.905402:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075658.417162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075658.929160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075658.929175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075658.929219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075659.249136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075659.249149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075659.249154:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075659.249163:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1364 00000100:00080000:3.0:1769075659.249178:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075659.249185:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075659.249218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913155072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075659.249247:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075659.249250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075659.249252:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075659.249257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1364 00000100:00080000:3.0:1769075659.249260:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075659.249263:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075659.249272:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075659.249286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913155200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075659.249301:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075659.249310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075659.249312:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075659.249318:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1364 00000100:00080000:3.0:1769075659.249322:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075659.249333:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075659.249343:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075659.249356:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913155328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075659.249363:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075659.249365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075659.249367:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075659.249372:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1364 00000100:00080000:3.0:1769075659.249377:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075659.249381:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075659.249388:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075659.249404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913155456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075659.249417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075659.249421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075659.249423:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075659.249429:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1364 00000100:00080000:3.0:1769075659.249434:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075659.249442:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075659.249450:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075659.249469:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913155584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075659.249495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075659.249501:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075659.249505:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075659.249516:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1364 00000100:00080000:3.0:1769075659.249525:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075659.249530:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075659.249537:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075659.249558:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913155712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075659.249578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075659.249582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075659.249585:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075659.249590:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1364 00000100:00080000:3.0:1769075659.249594:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075659.249599:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075659.249606:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075659.249639:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913155840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075659.441129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075659.953142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075659.953153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075659.953206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075660.465169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075660.977218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075660.977218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075660.977278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075661.489128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075662.001141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075662.001141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075662.001190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075662.513185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075663.025154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075663.025154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075663.025207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075663.537129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075664.049139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075664.049139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075664.049223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075664.369348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913155968:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075664.369424:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075664.369479:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075664/real 1769075664] req@ffff935002a62680 x1855008913155968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075664.369517:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913155968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075664.369563:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075664.369574:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075664.369585:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913155968:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075664.369599:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156096:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075664.369634:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075664.369652:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075664/real 1769075664] req@ffff935002a61880 x1855008913156096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075664.369667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913156096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075664.369677:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075664.369681:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075664.369684:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075664.369711:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075664.369732:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075664.369743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075664/real 1769075664] req@ffff935002a62a00 x1855008913156224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075664.369755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913156224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075664.369762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075664.369765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075664.369769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075664.369774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075664.369791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075664.369804:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075664/real 1769075664] req@ffff935002a60000 x1855008913156352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075664.369816:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913156352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075664.369835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075664.369839:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075664.369843:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075664.369848:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913156480:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075664.369868:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075664.369879:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075664/real 1769075664] req@ffff935002a61c00 x1855008913156480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075664.369891:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913156480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075664.369899:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075664.369902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075664.369906:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913156480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075664.369912:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913156608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075664.369929:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075664.369940:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075664/real 1769075664] req@ffff935002a62300 x1855008913156608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075664.369951:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913156608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075664.369958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075664.369962:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075664.369965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913156608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075664.369971:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913156736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075664.369986:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075664.369997:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075664/real 1769075664] req@ffff935002a61500 x1855008913156736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075664.370009:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913156736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075664.370017:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075664.370020:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075664.370024:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913156736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075664.561134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075665.073156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075665.073158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075665.073214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075665.585158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075666.097138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075666.097145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075666.097225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075666.609133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075667.121163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075667.121202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075667.121232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075667.633132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075668.145136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075668.145213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075668.145219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075668.657196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075669.169153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075669.169153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075669.169203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075669.489350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913156864:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075669.489425:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075669.489488:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075669/real 1769075669] req@ffff9350103f9180 x1855008913156864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075669.489514:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913156864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075669.489530:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075669.489535:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075669.489549:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913156864:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075669.489579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156992:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075669.489608:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075669.489621:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075669/real 1769075669] req@ffff9350103fad80 x1855008913156992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075669.489638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913156992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075669.489653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075669.489660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075669.489664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913156992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075669.489673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913157120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075669.489691:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075669.489702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075669/real 1769075669] req@ffff9350103f9500 x1855008913157120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075669.489730:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913157120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075669.489737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075669.489740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075669.489742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913157120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075669.489755:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913157248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075669.489767:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075669.489773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075669/real 1769075669] req@ffff9350103f8700 x1855008913157248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075669.489781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913157248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075669.489787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075669.489789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075669.489791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913157248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075669.489795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913157376:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075669.489804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075669.489809:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075669/real 1769075669] req@ffff9350103f9c00 x1855008913157376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075669.489816:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913157376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075669.489822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075669.489824:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075669.489826:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913157376:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075669.489837:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913157504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075669.489847:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075669.489853:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075669/real 1769075669] req@ffff9350103f9180 x1855008913157504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075669.489860:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913157504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075669.489872:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075669.489874:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075669.489876:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913157504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075669.489880:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913157632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075669.489890:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075669.489896:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075669/real 1769075669] req@ffff9350103fb800 x1855008913157632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075669.489904:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913157632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075669.489908:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075669.489910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075669.489912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913157632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075669.681150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075670.193094:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075670.193172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075670.193189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075670.705131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075671.217144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075671.217209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075671.729157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075672.241161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075672.241181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075672.753138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075673.265186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075673.265186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075673.777263:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075674.289174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075674.289174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075674.609283:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913157760:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075674.609350:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075674.609412:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075674/real 1769075674] req@ffff9350103fb480 x1855008913157760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075674.609431:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913157760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075674.609442:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075674.609447:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075674.609452:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913157760:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075674.609477:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913157888:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075674.609498:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075674.609511:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075674/real 1769075674] req@ffff9350103f8380 x1855008913157888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075674.609523:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913157888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075674.609533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075674.609536:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075674.609539:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913157888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075674.609545:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913158016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075674.609563:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075674.609572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075674/real 1769075674] req@ffff9350103faa00 x1855008913158016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075674.609580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913158016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075674.609585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075674.609587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075674.609589:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913158016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075674.609593:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913158144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075674.609610:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075674.609620:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075674/real 1769075674] req@ffff9350103f9f80 x1855008913158144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075674.609633:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913158144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075674.609642:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075674.609646:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075674.609649:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913158144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075674.609655:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913158272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075674.609672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075674.609680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075674/real 1769075674] req@ffff9350103fbb80 x1855008913158272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075674.609687:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913158272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075674.609694:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075674.609696:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075674.609699:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913158272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075674.609702:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913158400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075674.609711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075674.609717:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075674/real 1769075674] req@ffff9350103fb100 x1855008913158400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075674.609723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913158400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075674.609729:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075674.609731:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075674.609735:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913158400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075674.609755:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913158528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075674.609776:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075674.609787:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075674/real 1769075674] req@ffff9350103fb480 x1855008913158528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075674.609809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913158528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075674.609815:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075674.609817:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075674.609820:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913158528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075674.801130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075675.313151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075675.313152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075675.825151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075676.337146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075676.337154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075676.849107:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075677.361149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075677.361173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075677.873189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075678.385198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075678.385208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075678.897144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075679.409131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075679.409196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769075679.729186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075679.729197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075679.729201:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075679.729211:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1385 00000100:00080000:3.0:1769075679.729222:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075679.729236:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075679.729284:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913158656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075679.729310:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075679.729313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075679.729315:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075679.729318:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1385 00000100:00080000:3.0:1769075679.729321:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075679.729323:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075679.729332:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075679.729341:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913158784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075679.729349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075679.729351:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075679.729352:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075679.729355:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1385 00000100:00080000:3.0:1769075679.729357:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075679.729360:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075679.729364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075679.729372:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913158912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075679.729385:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075679.729387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075679.729389:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075679.729392:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1385 00000100:00080000:3.0:1769075679.729395:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075679.729397:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075679.729400:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075679.729408:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913159040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075679.729414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075679.729417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075679.729418:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075679.729421:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1385 00000100:00080000:3.0:1769075679.729433:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075679.729436:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075679.729439:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075679.729450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913159168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075679.729457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075679.729459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075679.729461:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075679.729464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1385 00000100:00080000:3.0:1769075679.729467:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075679.729469:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075679.729472:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075679.729479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913159296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075679.729486:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075679.729488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075679.729490:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075679.729496:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1385 00000100:00080000:3.0:1769075679.729501:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075679.729507:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075679.729514:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075679.729521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913159424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075679.921135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075680.433128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075680.433134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075680.945124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075681.457142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075681.457142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075681.969145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075682.481169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075682.481178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075682.993112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075683.505138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075683.505229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075684.017143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075684.529126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075684.529136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769075684.849133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075684.849143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075684.849147:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075684.849277:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1390 00000100:00080000:3.0:1769075684.849290:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075684.849306:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075684.849350:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913159552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075684.849380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075684.849384:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075684.849387:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075684.849391:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1390 00000100:00080000:3.0:1769075684.849395:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075684.849399:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075684.849411:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075684.849428:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913159680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075684.849448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075684.849452:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075684.849454:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075684.849460:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1390 00000100:00080000:3.0:1769075684.849464:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075684.849469:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075684.849474:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075684.849486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913159808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075684.849496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075684.849499:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075684.849501:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075684.849505:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1390 00000100:00080000:3.0:1769075684.849513:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075684.849520:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075684.849526:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075684.849538:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913159936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075684.849547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075684.849551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075684.849554:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075684.849559:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1390 00000100:00080000:3.0:1769075684.849566:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075684.849574:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075684.849584:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075684.849600:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913160064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075684.849626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075684.849629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075684.849632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075684.849637:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1390 00000100:00080000:3.0:1769075684.849644:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075684.849653:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075684.849662:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075684.849679:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913160192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075684.849689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075684.849692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075684.849693:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075684.849698:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1390 00000100:00080000:3.0:1769075684.849704:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075684.849712:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075684.849718:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075684.849732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913160320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075685.041124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075685.553150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075685.553163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075686.065144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075686.577208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075686.577221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075687.089122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075687.601123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075687.601155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075688.113104:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075688.625182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075688.625182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075689.137203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075689.649156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075689.649168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769075689.969160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.969170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075689.969175:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.969187:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1395 00000100:00080000:3.0:1769075689.969198:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.969205:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075689.969249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913160448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.969289:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.969293:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075689.969294:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.969298:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1395 00000100:00080000:3.0:1769075689.969301:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.969303:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075689.969313:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075689.969325:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913160576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.969333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.969336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075689.969337:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.969340:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1395 00000100:00080000:3.0:1769075689.969342:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.969344:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075689.969348:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075689.969356:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913160704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.969366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.969370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075689.969371:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.969373:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1395 00000100:00080000:3.0:1769075689.969375:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.969377:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075689.969380:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075689.969390:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913160832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.969396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.969399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075689.969400:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.969404:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1395 00000100:00080000:3.0:1769075689.969407:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.969410:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075689.969413:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075689.969419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913160960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.969425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.969427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075689.969429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.969431:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1395 00000100:00080000:3.0:1769075689.969443:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.969445:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075689.969448:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075689.969456:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913161088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.969465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.969467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075689.969469:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.969472:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1395 00000100:00080000:3.0:1769075689.969475:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.969477:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075689.969480:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075689.969486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913161216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075690.161157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075690.673176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075690.673176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075691.185149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075691.697148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075691.697148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075692.209174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075692.721172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075692.721173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075692.721222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075693.233124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075693.745138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075693.745167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075693.745182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075694.257204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075694.769189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075694.769189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075694.769267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075695.281175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075695.793148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075695.793148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075695.793191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075696.305120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075696.817168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075696.817168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075697.329162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075697.841160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075697.841160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075697.841227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075698.353166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075698.865231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075698.865236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075699.377140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075699.889186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075699.889268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075700.209150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075700.209161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075700.209166:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075700.209176:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1405 00000100:00080000:1.0:1769075700.209187:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075700.209193:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075700.209240:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913162240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075700.209273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075700.209277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075700.209279:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075700.209284:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1405 00000100:00080000:1.0:1769075700.209288:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075700.209292:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075700.209305:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075700.209318:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913162368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075700.209328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075700.209333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075700.209335:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075700.209341:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1405 00000100:00080000:1.0:1769075700.209344:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075700.209348:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075700.209354:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075700.209376:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913162496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075700.209389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075700.209393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075700.209395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075700.209400:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1405 00000100:00080000:1.0:1769075700.209404:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075700.209408:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075700.209413:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075700.209429:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913162624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075700.209443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075700.209447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075700.209449:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075700.209454:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1405 00000100:00080000:1.0:1769075700.209459:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075700.209464:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075700.209470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075700.209489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913162752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075700.209521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075700.209528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075700.209533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075700.209544:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1405 00000100:00080000:1.0:1769075700.209550:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075700.209556:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075700.209561:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075700.209576:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913162880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075700.209589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075700.209598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075700.209601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075700.209606:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1405 00000100:00080000:1.0:1769075700.209612:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075700.209618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075700.209622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075700.209634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913163008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075700.401111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075700.913130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075700.913177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075700.913255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075701.425124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075701.937241:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075701.937258:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075701.937291:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075702.449152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075702.961166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075702.961167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075702.961224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075703.473161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075703.985175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075703.985194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075703.985259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075704.497132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075705.009183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075705.009193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075705.009283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075705.329253:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075705.329272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075705.329281:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075705.329298:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1411 00000100:00080000:3.0:1769075705.329319:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075705.329332:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075705.329399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913163136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075705.329456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075705.329465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075705.329469:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075705.329478:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1411 00000100:00080000:3.0:1769075705.329486:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075705.329496:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075705.329515:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075705.329544:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913163264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075705.329583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075705.329590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075705.329593:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075705.329600:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1411 00000100:00080000:3.0:1769075705.329608:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075705.329614:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075705.329623:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075705.329643:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913163392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075705.329655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075705.329660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075705.329663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075705.329667:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1411 00000100:00080000:3.0:1769075705.329672:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075705.329676:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075705.329682:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075705.329705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913163520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075705.329720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075705.329726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075705.329728:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075705.329733:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1411 00000100:00080000:3.0:1769075705.329738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075705.329744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075705.329753:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075705.329776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913163648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075705.329792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075705.329800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075705.329807:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075705.329822:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1411 00000100:00080000:3.0:1769075705.329831:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075705.329870:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075705.329885:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075705.329913:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913163776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075705.329928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075705.329934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075705.329938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075705.329949:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1411 00000100:00080000:3.0:1769075705.329964:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075705.329968:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075705.329976:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075705.329996:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913163904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075705.521175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075706.033154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075706.033217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075706.545171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075707.057175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075707.057186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075707.569184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075708.081199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075708.081200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075708.593193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075709.105141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075709.105193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075709.617172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075710.129172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075710.129184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075710.449431:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913164032:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075710.449500:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075710.449549:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075710/real 1769075710] req@ffff935002a61180 x1855008913164032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075710.449578:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913164032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075710.449592:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075710.449597:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075710.449604:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913164032:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075710.449630:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913164160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075710.449652:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075710.449665:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075710/real 1769075710] req@ffff935002a63100 x1855008913164160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075710.449685:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913164160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075710.449694:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075710.449696:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075710.449701:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913164160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075710.449707:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913164288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075710.449720:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075710.449726:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075710/real 1769075710] req@ffff935002a61f80 x1855008913164288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075710.449734:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913164288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075710.449739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075710.449741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075710.449742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913164288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075710.449746:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913164416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075710.449786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075710.449816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075710/real 1769075710] req@ffff935002a60a80 x1855008913164416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075710.449832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913164416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075710.449843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075710.449847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075710.449851:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913164416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075710.449858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913164544:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075710.449879:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075710.449891:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075710/real 1769075710] req@ffff935002a61500 x1855008913164544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075710.449906:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913164544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075710.449916:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075710.449919:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075710.449923:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913164544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075710.449929:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913164672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075710.449947:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075710.449958:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075710/real 1769075710] req@ffff935002a62300 x1855008913164672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075710.449971:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913164672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075710.449981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075710.449984:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075710.449988:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913164672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075710.450011:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913164800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075710.450029:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075710.450040:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075710/real 1769075710] req@ffff935002a61c00 x1855008913164800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075710.450053:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913164800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075710.450098:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075710.450102:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075710.450107:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913164800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075710.641182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075711.153139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075711.153222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075711.665171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075712.177179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075712.177298:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075712.689144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075713.201145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075713.201145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075713.713116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075714.225136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075714.225136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075714.737193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075715.249123:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075715.249177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075715.569136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075715.569146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075715.569151:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075715.569160:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1421 00000100:00080000:3.0:1769075715.569171:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075715.569178:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075715.569212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913164928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075715.569261:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075715.569277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075715.569279:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075715.569285:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1421 00000100:00080000:3.0:1769075715.569291:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075715.569297:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075715.569310:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075715.569351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913165056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075715.569368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075715.569371:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075715.569373:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075715.569376:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1421 00000100:00080000:3.0:1769075715.569378:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075715.569381:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075715.569385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075715.569394:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913165184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075715.569400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075715.569431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075715.569434:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075715.569438:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1421 00000100:00080000:3.0:1769075715.569467:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075715.569473:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075715.569480:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075715.569506:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913165312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075715.569554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075715.569556:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075715.569557:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075715.569560:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1421 00000100:00080000:3.0:1769075715.569562:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075715.569565:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075715.569569:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075715.569580:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913165440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075715.569602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075715.569608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075715.569609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075715.569612:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1421 00000100:00080000:3.0:1769075715.569615:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075715.569618:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075715.569621:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075715.569630:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913165568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075715.569646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075715.569649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075715.569654:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075715.569659:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1421 00000100:00080000:3.0:1769075715.569682:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075715.569692:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075715.569697:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075715.569709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913165696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075715.761158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075716.273162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075716.273213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075716.785178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075717.297132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075717.297144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075717.809171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075718.321198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075718.321208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075718.833134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075719.345194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075719.345211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075719.345270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075719.857148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075720.369156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075720.369159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075720.689140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075720.689150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075720.689155:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075720.689163:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1426 00000100:00080000:3.0:1769075720.689175:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075720.689181:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075720.689249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913165824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075720.689275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075720.689279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075720.689282:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075720.689286:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1426 00000100:00080000:3.0:1769075720.689290:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075720.689294:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075720.689303:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075720.689315:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913165952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075720.689325:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075720.689328:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075720.689329:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075720.689333:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1426 00000100:00080000:3.0:1769075720.689335:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075720.689339:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075720.689343:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075720.689353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913166080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075720.689371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075720.689375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075720.689378:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075720.689390:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1426 00000100:00080000:3.0:1769075720.689406:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075720.689415:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075720.689424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075720.689452:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913166208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075720.689484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075720.689488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075720.689489:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075720.689493:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1426 00000100:00080000:3.0:1769075720.689497:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075720.689501:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075720.689505:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075720.689520:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913166336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075720.689528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075720.689532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075720.689534:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075720.689537:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1426 00000100:00080000:3.0:1769075720.689540:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075720.689548:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075720.689556:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075720.689568:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913166464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075720.689576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075720.689601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075720.689604:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075720.689614:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1426 00000100:00080000:3.0:1769075720.689631:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075720.689640:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075720.689650:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075720.689673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913166592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075720.881177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075721.393233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075721.393332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075721.905170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075722.417162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075722.417241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075722.929260:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075723.441188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075723.441189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000400:00000100:1.0:1769075723.761397:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769075723.761456:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075723.761478:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075723.761570:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075723.761598:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075723.761611:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075723.761627:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075723.761642:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769075723.761657:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075723.953138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075724.465167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075724.465168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075724.977158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075725.489166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075725.489319:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:3.0:1769075725.809169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075725.809187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075725.809194:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075725.809209:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1431 00000100:00080000:3.0:1769075725.809256:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075725.809263:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075725.809302:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913166720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075725.809332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075725.809336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075725.809339:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075725.809345:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1431 00000100:00080000:3.0:1769075725.809349:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075725.809355:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075725.809368:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075725.809390:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913166848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075725.809407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075725.809410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075725.809412:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075725.809418:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1431 00000100:00080000:3.0:1769075725.809423:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075725.809427:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075725.809434:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075725.809450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913166976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075725.809461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075725.809465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075725.809467:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075725.809472:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1431 00000100:00080000:3.0:1769075725.809478:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075725.809489:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075725.809514:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075725.809531:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913167104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075725.809542:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075725.809546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075725.809549:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075725.809556:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1431 00000100:00080000:3.0:1769075725.809561:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075725.809571:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075725.809580:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075725.809600:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913167232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075725.809621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075725.809629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075725.809633:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075725.809643:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1431 00000100:00080000:3.0:1769075725.809651:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075725.809661:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075725.809669:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075725.809682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913167360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075725.809703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075725.809709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075725.809711:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075725.809718:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1431 00000100:00080000:3.0:1769075725.809723:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075725.809730:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075725.809740:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075725.809755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913167488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075726.001175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075726.513184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075726.513184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075726.513252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075727.025186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075727.537136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075727.537264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075727.537364:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075728.049154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075728.561196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075728.561208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075729.073149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075729.585194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075729.585202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075730.097165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075730.609187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075730.609201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075730.929461:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913167616:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075730.929554:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075730.929626:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff935002a60000 x1855008913167616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075730.929655:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913167616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075730.929673:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075730.929680:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075730.929700:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913167616:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075730.929737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913167744:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075730.929771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075730.929791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff935002a62a00 x1855008913167744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075730.929813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913167744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075730.929829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075730.929835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075730.929841:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913167744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075730.929870:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913167872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075730.929908:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075730.929925:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff935002a61880 x1855008913167872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075730.929939:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913167872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075730.929949:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075730.929953:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075730.929956:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913167872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075730.929962:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913168000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075730.929982:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075730.929992:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff935002a62680 x1855008913168000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075730.930004:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913168000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075730.930013:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075730.930016:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075730.930019:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913168000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075730.930025:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913168128:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075730.930040:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075730.930082:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff935002a61c00 x1855008913168128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075730.930096:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913168128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075730.930106:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075730.930109:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075730.930113:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913168128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075730.930133:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913168256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075730.930153:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075730.930164:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff935002a62300 x1855008913168256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075730.930176:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913168256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075730.930185:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075730.930189:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075730.930192:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913168256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075730.930197:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913168384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075730.930212:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075730.930221:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff935002a61500 x1855008913168384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075730.930232:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913168384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075730.930241:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075730.930244:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075730.930247:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913168384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075731.121190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075731.633196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075731.633200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075732.145114:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075732.657218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075732.657219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075732.657331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075733.169173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075733.681112:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075733.681222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075733.681278:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075734.193133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075734.705197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075734.705221:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075734.705266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075735.217209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075735.729168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075735.729295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075735.729309:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:3.0:1769075736.049500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075736.049516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075736.049522:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075736.049535:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1441 00000100:00080000:3.0:1769075736.049551:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075736.049561:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075736.049612:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913168512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075736.049684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075736.049692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075736.049715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075736.049725:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1441 00000100:00080000:3.0:1769075736.049733:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075736.049750:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075736.049771:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075736.049827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913168640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075736.049849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075736.049854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075736.049857:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075736.049864:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1441 00000100:00080000:3.0:1769075736.049871:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075736.049878:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075736.049895:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075736.049921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913168768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075736.049958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075736.049964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075736.049967:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075736.049974:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1441 00000100:00080000:3.0:1769075736.049983:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075736.049992:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075736.050002:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075736.050034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913168896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075736.050053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075736.050059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075736.050064:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075736.050073:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1441 00000100:00080000:3.0:1769075736.050081:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075736.050144:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075736.050160:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075736.050194:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913169024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075736.050233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075736.050241:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075736.050246:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075736.050255:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1441 00000100:00080000:3.0:1769075736.050274:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075736.050289:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075736.050300:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075736.050361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913169152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075736.050391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075736.050403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075736.050412:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075736.050429:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1441 00000100:00080000:3.0:1769075736.050594:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075736.050611:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075736.050624:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075736.050659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913169280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075736.241199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075736.753184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075736.753185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075736.753374:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075737.265237:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075737.777155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075737.777315:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075737.777374:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075738.289204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075738.801271:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075738.801362:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075738.801412:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075739.313206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075739.825188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075739.825432:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075740.337157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075740.849180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075740.849220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075740.849237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075741.169290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075741.169301:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075741.169305:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075741.169315:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1446 00000100:00080000:1.0:1769075741.169327:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075741.169333:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075741.169376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913169408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075741.169404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075741.169409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075741.169411:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075741.169415:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1446 00000100:00080000:1.0:1769075741.169420:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075741.169425:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075741.169437:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075741.169457:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913169536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075741.169485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075741.169489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075741.169491:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075741.169498:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1446 00000100:00080000:1.0:1769075741.169501:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075741.169505:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075741.169522:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075741.169536:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913169664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075741.169544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075741.169547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075741.169549:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075741.169552:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1446 00000100:00080000:1.0:1769075741.169554:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075741.169559:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075741.169563:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075741.169574:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913169792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075741.169584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075741.169587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075741.169589:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075741.169593:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1446 00000100:00080000:1.0:1769075741.169595:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075741.169599:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075741.169602:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075741.169610:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913169920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075741.169618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075741.169620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075741.169622:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075741.169626:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1446 00000100:00080000:1.0:1769075741.169630:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075741.169636:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075741.169644:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075741.169665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913170048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075741.169704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075741.169708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075741.169714:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075741.169736:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1446 00000100:00080000:1.0:1769075741.169746:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075741.169754:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075741.169774:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075741.169798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913170176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075741.361188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075741.873180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075741.873261:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075741.873393:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075742.385159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075742.897228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075742.897254:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075743.409229:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075743.921259:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075743.921263:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075744.433178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075744.945202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075744.945236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075744.945285:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075744.945306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075745.457253:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075745.969288:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075745.969288:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075745.969333:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075745.969387:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075746.481199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075746.993202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075746.993235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075746.993296:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075747.505230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075748.017185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075748.017203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075748.017226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075748.529167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075749.041224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075749.041233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075749.041347:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075749.553190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075750.065241:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075750.065243:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075750.065381:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075750.577137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075751.089164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075751.089164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075751.601181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075752.113148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075752.113158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075752.113179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075752.625171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075753.137202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075753.137202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075753.649215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075754.161176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075754.161202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075754.161256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075754.673180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075755.185107:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075755.185121:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075755.185238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075755.697124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075756.209155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075756.209157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075756.721151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075757.233134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075757.233134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075757.745123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075758.257202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075758.257202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075758.769135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075759.281127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075759.281232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075759.793127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075760.305163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075760.305171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075760.305257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075760.817132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075761.329140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075761.329152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075761.329213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075761.649235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075761.649245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075761.649250:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075761.649260:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1467 00000100:00080000:1.0:1769075761.649271:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075761.649277:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075761.649320:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913172992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075761.649347:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075761.649350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075761.649352:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075761.649355:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1467 00000100:00080000:1.0:1769075761.649357:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075761.649360:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075761.649369:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075761.649380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913173120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075761.649388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075761.649390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075761.649392:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075761.649395:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1467 00000100:00080000:1.0:1769075761.649396:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075761.649399:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075761.649403:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075761.649410:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913173248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075761.649419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075761.649425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075761.649429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075761.649432:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1467 00000100:00080000:1.0:1769075761.649434:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075761.649436:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075761.649439:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075761.649448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913173376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075761.649455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075761.649457:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075761.649459:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075761.649461:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1467 00000100:00080000:1.0:1769075761.649463:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075761.649465:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075761.649468:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075761.649475:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913173504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075761.649481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075761.649483:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075761.649485:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075761.649487:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1467 00000100:00080000:1.0:1769075761.649490:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075761.649492:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075761.649496:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075761.649506:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913173632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075761.649512:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075761.649515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075761.649516:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075761.649520:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1467 00000100:00080000:1.0:1769075761.649523:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075761.649525:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075761.649527:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075761.649535:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913173760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075761.841163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075762.353158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075762.353167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075762.353210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075762.865137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075763.377131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075763.377138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075763.889141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075764.401132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075764.401146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075764.401227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075764.913169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075765.425125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075765.425129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075765.425161:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075765.937132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075766.449167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075766.449173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075766.449274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075766.961169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075767.473154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075767.473163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075767.473266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075767.985151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075768.497128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075768.497130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075768.497177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075769.009199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075769.521136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075769.521149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075769.521194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075770.033144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075770.545199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075770.545200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075770.545255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075771.057151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075771.569139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075771.569174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075771.569205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075771.889339:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913174784:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075771.889411:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075771.889452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff935002a60a80 x1855008913174784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075771.889481:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913174784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075771.889492:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075771.889500:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075771.889507:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913174784:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075771.889516:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913174912:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075771.889539:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075771.889559:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff935002a61f80 x1855008913174912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075771.889592:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913174912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075771.889611:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075771.889620:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075771.889628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913174912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075771.889653:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075771.889672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075771.889682:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff935002a63100 x1855008913175040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075771.889691:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913175040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075771.889696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075771.889698:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075771.889702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075771.889707:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075771.889740:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075771.889752:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff935002a61180 x1855008913175168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075771.889765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913175168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075771.889781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075771.889785:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075771.889789:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075771.889795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913175296:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075771.889816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075771.889828:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff935002a61500 x1855008913175296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075771.889841:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913175296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075771.889848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075771.889850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075771.889853:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913175296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075771.889857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913175424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075771.889872:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075771.889879:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff935002a62300 x1855008913175424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075771.889887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913175424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075771.889892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075771.889893:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075771.889896:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913175424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075771.889908:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913175552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075771.889919:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075771.889925:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff935002a61c00 x1855008913175552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075771.889932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913175552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075771.889938:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075771.889939:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075771.889941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913175552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075772.081168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075772.593158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075772.593289:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075773.105137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075773.617128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075773.617139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075774.129143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075774.641144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075774.641161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075775.153124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075775.665157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075775.665157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075776.177125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075776.689179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075776.689230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769075777.009280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075777.009293:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075777.009298:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075777.009309:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1482 00000100:00080000:1.0:1769075777.009324:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075777.009332:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075777.009382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913175680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075777.009417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075777.009421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075777.009423:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075777.009427:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1482 00000100:00080000:1.0:1769075777.009431:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075777.009434:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075777.009445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075777.009459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913175808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075777.009472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075777.009476:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075777.009479:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075777.009483:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1482 00000100:00080000:1.0:1769075777.009486:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075777.009495:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075777.009498:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913175680:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769075777.009502:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075777.009537:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913175936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075777.009550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075777.009555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075777.009558:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075777.009562:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1482 00000100:00080000:1.0:1769075777.009567:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075777.009572:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075777.009578:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075777.009593:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913176064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075777.009603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075777.009607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075777.009610:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075777.009615:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1482 00000100:00080000:1.0:1769075777.009621:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075777.009625:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075777.009632:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075777.009665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075777.009683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913176192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075777.009699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075777.009704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075777.009707:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075777.009713:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1482 00000100:00080000:1.0:1769075777.009719:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075777.009724:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075777.009731:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075777.009740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075777/real 1769075777] req@ffff9350062a5f80 x1855008913175680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075777.009750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913176320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075777.009778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075777.009790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075777.009794:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075777.009804:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913175680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075777.009806:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1482 00000100:00080000:1.0:1769075777.009821:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075777.009833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075777.009837:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:3.0:1769075777.009847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769075777.009856:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075777.009857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913175680:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075777.009879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769075777.009897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913176448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075777.009934:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075777.009971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075777/real 1769075777] req@ffff9350062a4380 x1855008913175808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075777.010000:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913175808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075777.010016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075777.010021:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075777.010026:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075777.010118:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075777.010152:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075777.010169:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075777/real 1769075777] req@ffff9350062a5500 x1855008913175936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075777.010184:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913175936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075777.010196:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075777.010200:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075777.010204:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913175936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075777.010212:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075777.010231:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075777.010242:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075777/real 1769075777] req@ffff9350062a5c00 x1855008913176064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075777.010256:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913176064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075777.010283:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075777.010286:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075777.010289:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075777.010295:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913176192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075777.010334:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075777.010350:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075777/real 1769075777] req@ffff9350062a5180 x1855008913176192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075777.010371:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913176192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075777.010385:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075777.010391:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075777.010397:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913176192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075777.010407:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913176320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075777.010435:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075777.010448:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075777/real 1769075777] req@ffff9350062a5880 x1855008913176320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075777.010461:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913176320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075777.010470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075777.010473:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075777.010476:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913176320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075777.010482:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913176448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075777.010498:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075777.010506:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075777/real 1769075777] req@ffff9350062a4a80 x1855008913176448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075777.010518:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913176448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075777.010526:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075777.010528:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075777.010531:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913176448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075777.201210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075777.713236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075777.713248:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075777.713386:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075778.225184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075778.737196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075778.737250:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075778.737254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075779.249144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075779.761145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075779.761197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075779.761262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075780.273161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075780.785203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075780.785224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075780.785256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075781.297204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075781.809265:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075781.809265:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075781.809343:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075782.129467:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913176576:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075782.129614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075782.129693:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075782/real 1769075782] req@ffff935002a62680 x1855008913176576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075782.129751:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913176576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075782.129843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075782.129876:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075782.129897:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913176576:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075782.129937:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176704:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075782.129976:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075782.129996:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075782/real 1769075782] req@ffff935002a61880 x1855008913176704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075782.130084:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913176704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075782.130097:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075782.130103:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075782.130107:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075782.130118:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075782.130159:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075782.130184:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075782/real 1769075782] req@ffff935002a62a00 x1855008913176832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075782.130254:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913176832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075782.130270:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075782.130275:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075782.130278:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075782.130285:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075782.130301:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075782.130309:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075782/real 1769075782] req@ffff935002a60000 x1855008913176960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075782.130319:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913176960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075782.130326:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075782.130329:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075782.130331:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913176960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075782.130337:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177088:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075782.130351:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075782.130359:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075782/real 1769075782] req@ffff935002a61c00 x1855008913177088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075782.130376:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913177088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075782.130387:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075782.130392:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075782.130397:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075782.130426:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075782.130456:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075782.130473:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075782/real 1769075782] req@ffff935002a62300 x1855008913177216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075782.130487:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913177216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075782.130495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075782.130498:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075782.130502:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075782.130508:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075782.130527:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075782.130536:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075782/real 1769075782] req@ffff935002a61500 x1855008913177344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075782.130546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913177344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075782.130602:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075782.130607:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075782.130611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075782.321261:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075782.833140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075782.833229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075782.833240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075783.345134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075783.857209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075783.857244:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075784.369240:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075784.881294:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075784.881369:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075785.393170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075785.905153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075785.905153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075786.417296:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075786.929154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075786.929235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769075787.249256:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075787.249270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075787.249276:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075787.249287:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1492 00000100:00080000:1.0:1769075787.249301:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075787.249309:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075787.249350:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913177472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075787.249381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075787.249384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075787.249386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075787.249389:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1492 00000100:00080000:1.0:1769075787.249392:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075787.249396:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075787.249405:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075787.249414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913177600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075787.249431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075787.249435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075787.249437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075787.249442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1492 00000100:00080000:1.0:1769075787.249444:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075787.249447:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075787.249452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075787.249483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913177728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075787.249505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075787.249513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075787.249516:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075787.249517:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913177472:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075787.249528:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1492 00000100:00080000:1.0:1769075787.249539:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075787.249544:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075787.249549:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075787.249564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913177856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075787.249571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075787.249575:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075787.249577:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075787.249580:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1492 00000100:00080000:1.0:1769075787.249583:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075787.249586:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075787.249590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075787.249600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913177984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075787.249605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075787.249608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075787.249612:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075787.249615:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1492 00000100:00080000:1.0:1769075787.249617:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075787.249619:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075787.249623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075787.249631:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913178112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075787.249638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075787.249641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075787.249643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075787.249645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1492 00000100:00080000:1.0:1769075787.249652:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075787.249654:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075787.249659:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075787.249676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913178240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075787.249689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913177600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075787.249768:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913177728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075787.249803:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913177856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075787.249826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075787.249853:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913178112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075787.249877:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913178240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075787.441173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075787.953168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075787.953212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075787.953223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075788.465200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075788.977135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075788.977174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075789.489164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075790.001164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075790.321714:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075787/real 1769075790] req@ffff9350062a5180 x1855008913177728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075790.321745:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913177728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075790.321761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075790.321767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075790.321773:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913177728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075790.321783:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075787/real 1769075790] req@ffff9350062a5c00 x1855008913177856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075790.321797:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913177856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075790.321805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075790.321809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075790.321813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913177856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075790.321820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075787/real 1769075790] req@ffff9350062a5500 x1855008913177984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075790.321833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913177984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075790.321842:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075790.321846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075790.321849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913177984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075790.321856:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075787/real 1769075790] req@ffff9350062a4380 x1855008913178112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075790.321868:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913178112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075790.321876:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075790.321879:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075790.321882:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913178112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075790.321890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075787/real 1769075790] req@ffff9350062a5f80 x1855008913178240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075790.321902:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913178240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075790.321918:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075790.321921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075790.321925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913178240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075790.321933:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075787/real 1769075790] req@ffff9350062a4a80 x1855008913177472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075790.321946:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913177472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075790.321954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075790.321957:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075790.321961:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913177472:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075790.321969:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075787/real 1769075790] req@ffff9350062a5880 x1855008913177600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075790.321980:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913177600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075790.321989:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075790.321992:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075790.321995:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913177600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075790.513210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075791.025168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075791.025237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075791.025293:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075791.537166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075792.049163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075792.049221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075792.049254:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075792.369604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913178368:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075792.369750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075792.369835:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075792/real 1769075792] req@ffff9350103fb100 x1855008913178368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075792.369893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913178368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075792.369948:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075792.369963:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075792.369975:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913178368:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075792.370026:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913178496:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075792.370163:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075792.370192:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075792/real 1769075792] req@ffff9350103fbb80 x1855008913178496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075792.370217:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913178496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075792.370233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075792.370238:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075792.370244:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913178496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075792.370256:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913178624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075792.370292:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075792.370311:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075792/real 1769075792] req@ffff9350103f9f80 x1855008913178624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075792.370503:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913178624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075792.370518:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075792.370524:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075792.370529:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913178624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075792.370543:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913178752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075792.370581:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075792.370601:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075792/real 1769075792] req@ffff9350103faa00 x1855008913178752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075792.370621:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913178752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075792.370635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075792.370640:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075792.370645:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913178752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075792.370655:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913178880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075792.370687:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075792.370707:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075792/real 1769075792] req@ffff9350103f8380 x1855008913178880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075792.370728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913178880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075792.370741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075792.370746:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075792.370752:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913178880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075792.370762:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075792.370794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075792.370813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075792/real 1769075792] req@ffff9350103fb480 x1855008913179008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075792.370835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913179008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075792.370849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075792.370854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075792.370859:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075792.370895:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075792.370927:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075792.370946:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075792/real 1769075792] req@ffff9350103fad80 x1855008913179136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075792.370966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913179136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075792.370979:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075792.370985:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075792.370991:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075792.561173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075793.073234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075793.073275:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075793.073320:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075793.585180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075794.097151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075794.097152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075794.097204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075794.609162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075795.121134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075795.121134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075795.121202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075795.633156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075796.145174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075796.145231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075796.145239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075796.657212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075797.169198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075797.169212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075797.169367:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075797.489505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913179264:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075797.489616:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075797.489693:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075797/real 1769075797] req@ffff9350103f9500 x1855008913179264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075797.489757:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913179264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075797.489805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075797.489823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075797.489840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913179264:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075797.489859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913179392:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075797.489897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075797.489916:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075797/real 1769075797] req@ffff9350103f8700 x1855008913179392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075797.489933:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913179392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075797.489944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075797.489949:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075797.489954:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913179392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075797.489989:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913179520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075797.490010:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075797.490023:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075797/real 1769075797] req@ffff9350103f9c00 x1855008913179520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075797.490037:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913179520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075797.490094:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075797.490099:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075797.490104:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913179520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075797.490111:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913179648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075797.490134:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075797.490148:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075797/real 1769075797] req@ffff9350103f9180 x1855008913179648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075797.490162:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913179648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075797.490172:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075797.490176:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075797.490180:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913179648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075797.490186:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179776:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075797.490204:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075797.490214:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075797/real 1769075797] req@ffff9350103fb800 x1855008913179776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075797.490228:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913179776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075797.490239:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075797.490242:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075797.490246:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075797.490252:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075797.490269:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075797.490279:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075797/real 1769075797] req@ffff9350103fad80 x1855008913179904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075797.490321:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913179904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075797.490332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075797.490336:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075797.490340:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913179904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075797.490348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075797.490375:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075797.490389:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075797/real 1769075797] req@ffff9350103fb480 x1855008913180032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075797.490403:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913180032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075797.490413:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075797.490417:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075797.490432:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075797.681155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075798.193171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075798.193214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075798.193330:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075798.705165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075799.217176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075799.217190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075799.217313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075799.729182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075800.241158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075800.241173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075800.241277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075800.753157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075801.265198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075801.265215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075802.289203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075802.289204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075802.289285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075802.609327:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913180160:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075802.609411:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075802.609473:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075802/real 1769075802] req@ffff9350103f8380 x1855008913180160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075802.609516:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913180160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075802.609527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075802.609532:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075802.609536:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913180160:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075802.609561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913180288:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075802.609577:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075802.609586:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075802/real 1769075802] req@ffff9350103faa00 x1855008913180288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075802.609607:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913180288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075802.609617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075802.609621:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075802.609625:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913180288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075802.609631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913180416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075802.609653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075802.609663:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075802/real 1769075802] req@ffff9350103f9f80 x1855008913180416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075802.609671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913180416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075802.609676:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075802.609678:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075802.609688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913180416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075802.609692:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913180544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075802.609705:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075802.609711:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075802/real 1769075802] req@ffff9350103fbb80 x1855008913180544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075802.609719:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913180544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075802.609729:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075802.609731:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075802.609733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913180544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075802.609736:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180672:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075802.609746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075802.609751:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075802/real 1769075802] req@ffff9350103fb100 x1855008913180672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075802.609758:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913180672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075802.609763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075802.609765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075802.609767:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075802.609770:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075802.609779:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075802.609784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075802/real 1769075802] req@ffff9350103fb480 x1855008913180800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075802.609791:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913180800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075802.609796:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075802.609798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075802.609801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075802.609804:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075802.609814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075802.609820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075802/real 1769075802] req@ffff9350103fad80 x1855008913180928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075802.609826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913180928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075802.609832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075802.609833:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075802.609835:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913180928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075803.313185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075803.313185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075803.313344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075804.337298:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075804.337304:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075804.337372:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075805.361234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075805.361257:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075805.361288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075806.385137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075806.385286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075806.385310:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075807.409207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075807.409207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075807.409285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075807.729350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913181056:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075807.729426:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075807.729488:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075807/real 1769075807] req@ffff935002a61180 x1855008913181056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075807.729523:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913181056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075807.729555:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075807.729568:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075807.729579:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913181056:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075807.729614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913181184:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075807.729644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075807.729659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075807/real 1769075807] req@ffff935002a63100 x1855008913181184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075807.729674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913181184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075807.729683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075807.729685:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075807.729688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913181184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075807.729693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913181312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075807.729710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075807.729718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075807/real 1769075807] req@ffff935002a61f80 x1855008913181312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075807.729727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913181312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075807.729733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075807.729735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075807.729737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913181312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075807.729741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913181440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075807.729752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075807.729758:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075807/real 1769075807] req@ffff935002a60a80 x1855008913181440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075807.729765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913181440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075807.729771:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075807.729773:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075807.729775:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913181440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075807.729778:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913181568:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075807.729788:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075807.729794:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075807/real 1769075807] req@ffff935002a61500 x1855008913181568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075807.729801:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913181568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075807.729806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075807.729807:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075807.729809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913181568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075807.729813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913181696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075807.729822:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075807.729829:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075807/real 1769075807] req@ffff935002a62300 x1855008913181696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075807.729837:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913181696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075807.729843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075807.729844:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075807.729846:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913181696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075807.729850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913181824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075807.729859:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075807.729865:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075807/real 1769075807] req@ffff935002a61c00 x1855008913181824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075807.729872:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913181824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075807.729878:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075807.729880:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075807.729881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913181824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075808.433156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075808.433235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075808.433254:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075808.433359:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075809.457246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075809.457303:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075809.457454:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075810.481227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075810.481252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075810.481307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075811.505258:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075811.505272:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075811.505370:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075812.529230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075812.529230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075812.529295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075812.849211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.849224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075812.849230:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.849243:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1518 00000100:00080000:3.0:1769075812.849258:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.849265:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075812.849325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913181952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075812.849372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.849377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075812.849379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.849385:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1518 00000100:00080000:3.0:1769075812.849390:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.849395:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075812.849408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075812.849426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913182080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075812.849439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.849443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075812.849445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.849450:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1518 00000100:00080000:3.0:1769075812.849454:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.849457:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075812.849461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075812.849474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913182208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075812.849483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.849487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075812.849488:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.849491:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1518 00000100:00080000:3.0:1769075812.849504:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.849507:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075812.849511:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075812.849527:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913182336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075812.849538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.849542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075812.849544:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.849547:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1518 00000100:00080000:3.0:1769075812.849551:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.849554:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075812.849559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075812.849575:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913182464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075812.849621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.849625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075812.849627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.849634:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1518 00000100:00080000:3.0:1769075812.849641:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.849649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075812.849658:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075812.849666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913182592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075812.849678:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.849683:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075812.849688:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.849699:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1518 00000100:00080000:3.0:1769075812.849706:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.849712:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075812.849716:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075812.849724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913182720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075813.553199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075813.553199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075813.553363:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075814.577144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075814.577240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075815.601204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075815.601220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075815.601324:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075816.625250:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075816.625260:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075816.625318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075817.649237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075817.649245:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075817.649307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075817.969219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.969234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075817.969240:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.969255:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1523 00000100:00080000:3.0:1769075817.969282:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.969292:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075817.969348:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913182848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.969403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.969411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075817.969414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.969424:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1523 00000100:00080000:3.0:1769075817.969431:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.969438:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075817.969458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075817.969488:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913182976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.969518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.969528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075817.969533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.969544:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1523 00000100:00080000:3.0:1769075817.969553:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.969559:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075817.969569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075817.969589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913183104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.969603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.969608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075817.969616:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.969631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1523 00000100:00080000:3.0:1769075817.969641:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.969651:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075817.969661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075817.969686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913183232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.969714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.969729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075817.969737:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.969759:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1523 00000100:00080000:3.0:1769075817.969769:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.969780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075817.969793:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075817.969831:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913183360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.969846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.969854:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075817.969858:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.969868:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1523 00000100:00080000:3.0:1769075817.969875:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.969884:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075817.969899:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075817.969915:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913183488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.969928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.969933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075817.969940:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.969991:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1523 00000100:00080000:3.0:1769075817.969999:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.970008:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075817.970019:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075817.970108:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913183616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075818.673194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075818.673194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075818.673249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075819.697225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075819.697230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075819.697281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075820.721239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075820.721239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075820.721283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075821.745182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075821.745197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075821.745228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075822.769169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075822.769174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075822.769212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075823.089465:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913183744:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075823.089579:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075823.089632:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075823/real 1769075823] req@ffff935002a60000 x1855008913183744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075823.089667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913183744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075823.089686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075823.089695:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075823.089707:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913183744:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075823.089745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913183872:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075823.089809:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075823.089834:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075823/real 1769075823] req@ffff935002a62a00 x1855008913183872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075823.089892:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913183872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075823.089917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075823.089925:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075823.089941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913183872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075823.089958:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913184000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075823.090005:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075823.090019:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075823/real 1769075823] req@ffff935002a61880 x1855008913184000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075823.090030:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913184000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075823.090037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075823.090040:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075823.090042:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913184000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075823.090046:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913184128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075823.090082:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075823.090090:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075823/real 1769075823] req@ffff935002a62680 x1855008913184128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075823.090098:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913184128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075823.090105:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075823.090107:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075823.090109:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913184128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075823.090125:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913184256:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075823.090136:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075823.090143:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075823/real 1769075823] req@ffff935002a61c00 x1855008913184256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075823.090151:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913184256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075823.090156:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075823.090158:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075823.090160:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913184256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075823.090164:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913184384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075823.090174:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075823.090180:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075823/real 1769075823] req@ffff935002a62300 x1855008913184384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075823.090188:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913184384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075823.090194:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075823.090196:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075823.090199:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913184384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075823.090202:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913184512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075823.090211:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075823.090217:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075823/real 1769075823] req@ffff935002a61500 x1855008913184512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075823.090223:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913184512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075823.090228:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075823.090231:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075823.090233:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913184512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075823.793172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075823.793212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075823.793301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075824.817179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075824.817233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075824.817251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075825.841288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075825.841315:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075826.865167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075826.865318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075827.889174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075827.889193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075827.889215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075828.209203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075828.209213:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075828.209217:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075828.209227:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1533 00000100:00080000:3.0:1769075828.209237:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075828.209243:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075828.209281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913184640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075828.209319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075828.209322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075828.209324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075828.209327:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1533 00000100:00080000:3.0:1769075828.209330:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075828.209333:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075828.209343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075828.209353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913184768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075828.209366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075828.209371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075828.209373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075828.209378:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1533 00000100:00080000:3.0:1769075828.209383:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075828.209387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075828.209393:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075828.209407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913184896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075828.209419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075828.209422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075828.209424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075828.209431:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1533 00000100:00080000:3.0:1769075828.209438:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075828.209442:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075828.209447:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075828.209458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913185024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075828.209465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075828.209468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075828.209470:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075828.209473:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1533 00000100:00080000:3.0:1769075828.209476:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075828.209489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075828.209496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075828.209508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913185152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075828.209541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075828.209545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075828.209549:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075828.209554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1533 00000100:00080000:3.0:1769075828.209561:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075828.209566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075828.209574:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075828.209581:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913185280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075828.209595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075828.209599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075828.209602:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075828.209605:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1533 00000100:00080000:3.0:1769075828.209607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075828.209610:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075828.209616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075828.209625:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913185408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075828.913154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075828.913155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075828.913189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075829.937139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075829.937172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075830.961208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075830.961429:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075831.985195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075831.985310:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075831.985327:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075833.009181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075833.009231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075833.009250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075833.009347:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075833.329189:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075833.329204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075833.329210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075833.329224:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1539 00000100:00080000:3.0:1769075833.329242:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075833.329253:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075833.329317:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913185536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075833.329370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075833.329378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075833.329382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075833.329393:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1539 00000100:00080000:3.0:1769075833.329401:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075833.329411:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075833.329427:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075833.329453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913185664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075833.329481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075833.329486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075833.329489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075833.329496:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1539 00000100:00080000:3.0:1769075833.329501:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075833.329505:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075833.329513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075833.329531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913185792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075833.329543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075833.329548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075833.329552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075833.329556:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1539 00000100:00080000:3.0:1769075833.329561:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075833.329566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075833.329577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075833.329597:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913185920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075833.329612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075833.329617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075833.329620:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075833.329626:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1539 00000100:00080000:3.0:1769075833.329632:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075833.329641:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075833.329655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075833.329680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913186048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075833.329731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075833.329744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075833.329750:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075833.329764:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1539 00000100:00080000:3.0:1769075833.329773:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075833.329781:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075833.329793:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075833.329815:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913186176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075833.329834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075833.329838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075833.329841:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075833.329846:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1539 00000100:00080000:3.0:1769075833.329850:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075833.329855:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075833.329861:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075833.329880:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913186304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075834.033205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075834.033228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075834.033259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075835.057222:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075835.057256:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075835.057335:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075836.081146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075836.081167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075836.081215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075837.105226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075837.105245:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075837.105305:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075838.129204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075838.129211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075838.129300:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075838.449152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075838.449164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075838.449169:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075838.449182:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1544 00000100:00080000:3.0:1769075838.449197:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075838.449205:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075838.449285:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913186432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075838.449334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075838.449341:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075838.449345:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075838.449353:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1544 00000100:00080000:3.0:1769075838.449362:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075838.449371:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075838.449387:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075838.449406:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913186560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075838.449442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075838.449448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075838.449450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075838.449456:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1544 00000100:00080000:3.0:1769075838.449460:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075838.449465:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075838.449479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075838.449493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913186688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075838.449505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075838.449509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075838.449511:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075838.449531:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1544 00000100:00080000:3.0:1769075838.449543:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075838.449546:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075838.449551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075838.449563:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913186816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075838.449578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075838.449598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075838.449600:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075838.449635:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1544 00000100:00080000:3.0:1769075838.449640:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075838.449644:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075838.449666:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075838.449681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913186944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075838.449691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075838.449698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075838.449702:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075838.449711:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1544 00000100:00080000:3.0:1769075838.449719:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075838.449730:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075838.449743:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075838.449764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913187072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075838.449812:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075838.449827:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075838.449831:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075838.449837:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1544 00000100:00080000:3.0:1769075838.449844:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075838.449865:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075838.449872:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075838.449891:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913187200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075839.153195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075839.153195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075840.177147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075840.177155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075841.201150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075841.201177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075841.201214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075842.225166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075842.225166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075843.249205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075843.249214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075843.569172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075843.569182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075843.569186:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075843.569195:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1549 00000100:00080000:3.0:1769075843.569205:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075843.569213:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075843.569246:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913187328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075843.569270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075843.569273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075843.569274:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075843.569277:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1549 00000100:00080000:3.0:1769075843.569280:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075843.569283:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075843.569292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075843.569302:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913187456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075843.569316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075843.569320:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075843.569321:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075843.569325:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1549 00000100:00080000:3.0:1769075843.569328:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075843.569331:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075843.569337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075843.569350:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913187584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075843.569360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075843.569363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075843.569365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075843.569370:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1549 00000100:00080000:3.0:1769075843.569374:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075843.569378:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075843.569390:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075843.569407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913187712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075843.569417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075843.569423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075843.569427:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075843.569438:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1549 00000100:00080000:3.0:1769075843.569444:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075843.569461:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075843.569469:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075843.569486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913187840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075843.569563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075843.569570:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075843.569573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075843.569579:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1549 00000100:00080000:3.0:1769075843.569601:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075843.569620:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075843.569632:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075843.569650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913187968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075843.569672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075843.569680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075843.569685:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075843.569711:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1549 00000100:00080000:3.0:1769075843.569734:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075843.569744:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075843.569764:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075843.569789:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913188096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075844.273174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075844.273235:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075845.297195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075845.297212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075846.321208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075846.321264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075846.321289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075847.345103:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075847.345152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075847.345168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075848.369163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075848.369196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075848.369233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075848.689171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075848.689181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075848.689185:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075848.689195:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1554 00000100:00080000:1.0:1769075848.689206:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075848.689213:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075848.689255:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913188224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075848.689280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075848.689284:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075848.689285:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075848.689288:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1554 00000100:00080000:1.0:1769075848.689298:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075848.689302:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075848.689314:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075848.689330:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913188352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075848.689341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075848.689345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075848.689348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075848.689353:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1554 00000100:00080000:1.0:1769075848.689357:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075848.689365:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075848.689371:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075848.689384:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913188480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075848.689392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075848.689395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075848.689396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075848.689399:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1554 00000100:00080000:1.0:1769075848.689403:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075848.689405:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075848.689409:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075848.689416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913188608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075848.689421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075848.689426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075848.689428:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075848.689449:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1554 00000100:00080000:1.0:1769075848.689452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075848.689456:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075848.689460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075848.689471:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913188736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075848.689478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075848.689480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075848.689482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075848.689485:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1554 00000100:00080000:1.0:1769075848.689488:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075848.689490:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075848.689496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075848.689511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913188864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075848.689520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075848.689529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075848.689532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075848.689538:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1554 00000100:00080000:1.0:1769075848.689540:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075848.689547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075848.689555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075848.689565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913188992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075849.393161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075849.393190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075850.417137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075850.417185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075851.441185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075852.465119:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075852.465152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075853.489186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075853.489192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:1.0:1769075853.809224:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075853.809238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075853.809244:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075853.809256:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1559 00000100:00080000:1.0:1769075853.809271:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075853.809278:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075853.809332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913189120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075853.809416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075853.809420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075853.809423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075853.809428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1559 00000100:00080000:1.0:1769075853.809432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075853.809437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075853.809450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075853.809468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913189248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075853.809480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075853.809484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075853.809486:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075853.809491:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1559 00000100:00080000:1.0:1769075853.809494:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075853.809497:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075853.809503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075853.809519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913189376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075853.809532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075853.809536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075853.809538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075853.809542:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1559 00000100:00080000:1.0:1769075853.809546:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075853.809549:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075853.809554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075853.809571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913189504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075853.809580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075853.809584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075853.809587:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075853.809591:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1559 00000100:00080000:1.0:1769075853.809604:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075853.809609:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075853.809614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075853.809626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913189632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075853.809636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075853.809640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075853.809642:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075853.809647:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1559 00000100:00080000:1.0:1769075853.809652:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075853.809656:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075853.809662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075853.809678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913189760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075853.809691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075853.809696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075853.809698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075853.809704:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1559 00000100:00080000:1.0:1769075853.809709:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075853.809714:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075853.809720:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075853.809749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913189888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075854.513153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075854.513154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075855.537150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075855.537150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075856.561250:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075856.561256:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000400:00000100:3.0:1769075856.881503:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769075856.881566:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075856.881591:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075856.881647:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075856.881664:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075856.881694:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075856.881705:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075856.881717:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769075856.881728:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769075857.585129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075858.609174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075858.609216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:1.0:1769075858.929149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075858.929159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075858.929163:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.929191:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1564 00000100:00080000:1.0:1769075858.929203:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075858.929210:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075858.929282:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913190016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075858.929310:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075858.929313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075858.929315:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.929318:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1564 00000100:00080000:1.0:1769075858.929321:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075858.929324:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075858.929334:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075858.929346:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913190144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075858.929354:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075858.929357:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075858.929359:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.929362:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1564 00000100:00080000:1.0:1769075858.929364:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075858.929367:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075858.929373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075858.929388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913190272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075858.929405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075858.929408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075858.929411:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.929416:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1564 00000100:00080000:1.0:1769075858.929421:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075858.929426:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075858.929432:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075858.929448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913190400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075858.929458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075858.929462:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075858.929464:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.929470:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1564 00000100:00080000:1.0:1769075858.929479:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075858.929487:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075858.929494:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075858.929509:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913190528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075858.929521:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075858.929524:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075858.929526:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.929532:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1564 00000100:00080000:1.0:1769075858.929536:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075858.929545:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075858.929556:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075858.929577:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913190656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075858.929607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075858.929626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075858.929632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.929645:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1564 00000100:00080000:1.0:1769075858.929653:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075858.929663:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075858.929670:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075858.929687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913190784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075859.633242:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075859.633310:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075860.657217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075860.657220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075861.681210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075861.681237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075862.705177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075862.705218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075863.729251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075863.729261:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075864.049341:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075864.049362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075864.049371:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075864.049391:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1569 00000100:00080000:3.0:1769075864.049411:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075864.049424:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075864.049512:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913190912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075864.049558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075864.049563:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075864.049566:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075864.049570:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1569 00000100:00080000:3.0:1769075864.049575:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075864.049579:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075864.049592:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075864.049609:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913191040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075864.049620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075864.049623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075864.049625:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075864.049632:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1569 00000100:00080000:3.0:1769075864.049635:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075864.049639:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075864.049645:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075864.049670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913191168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075864.049683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075864.049686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075864.049688:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075864.049692:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1569 00000100:00080000:3.0:1769075864.049696:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075864.049700:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075864.049705:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075864.049719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913191296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075864.049729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075864.049734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075864.049736:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075864.049740:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1569 00000100:00080000:3.0:1769075864.049744:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075864.049750:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075864.049761:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075864.049779:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913191424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075864.049810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075864.049814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075864.049817:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075864.049823:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1569 00000100:00080000:3.0:1769075864.049830:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075864.049836:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075864.049845:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075864.049869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913191552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075864.049885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075864.049911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075864.049917:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075864.049931:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1569 00000100:00080000:3.0:1769075864.049937:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075864.049946:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075864.049956:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075864.049976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913191680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075864.753209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075864.753256:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075865.777198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075865.777211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075866.801213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075866.801213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075867.825171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075867.825173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075868.849206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075868.849206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075869.169309:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913191808:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075869.169393:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075869.169442:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075869/real 1769075869] req@ffff935002a60a80 x1855008913191808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075869.169467:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913191808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075869.169483:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075869.169491:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075869.169495:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913191808:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075869.169523:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913191936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075869.169550:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075869.169567:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075869/real 1769075869] req@ffff935002a61f80 x1855008913191936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075869.169586:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913191936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075869.169597:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075869.169601:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075869.169605:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913191936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075869.169613:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075869.169635:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075869.169644:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075869/real 1769075869] req@ffff935002a63100 x1855008913192064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075869.169658:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913192064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075869.169664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075869.169666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075869.169668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075869.169672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075869.169705:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075869.169719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075869/real 1769075869] req@ffff935002a61180 x1855008913192192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075869.169733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913192192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075869.169743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075869.169764:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075869.169767:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075869.169773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913192320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075869.169789:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075869.169797:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075869/real 1769075869] req@ffff935002a61500 x1855008913192320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075869.169837:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913192320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075869.169852:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075869.169855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075869.169857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913192320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075869.169875:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913192448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075869.169909:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075869.169929:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075869/real 1769075869] req@ffff935002a62300 x1855008913192448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075869.169943:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913192448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075869.169953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075869.169974:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075869.169978:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913192448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075869.169985:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913192576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075869.170002:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075869.170012:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075869/real 1769075869] req@ffff935002a61c00 x1855008913192576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075869.170041:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913192576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075869.170046:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075869.170048:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075869.170051:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913192576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075869.873155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075870.897172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075870.897262:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075871.921248:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075871.921267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075872.945220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075872.945222:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075872.945366:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075873.969349:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075873.969377:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075873.969436:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769075874.289286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075874.289300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075874.289305:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075874.289318:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1579 00000100:00080000:1.0:1769075874.289332:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075874.289340:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075874.289393:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913192704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075874.289430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075874.289434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075874.289436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075874.289441:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1579 00000100:00080000:1.0:1769075874.289444:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075874.289448:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075874.289460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075874.289477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913192832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075874.289488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075874.289492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075874.289493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075874.289497:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1579 00000100:00080000:1.0:1769075874.289501:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075874.289505:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075874.289509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075874.289521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913192960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075874.289531:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913192704:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769075874.289536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075874.289557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075874.289559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075874.289564:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1579 00000100:00080000:1.0:1769075874.289567:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075874.289570:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075874.289575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075874.289590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913193088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075874.289599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075874.289603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075874.289604:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075874.289608:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1579 00000100:00080000:1.0:1769075874.289613:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075874.289616:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075874.289621:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075874.289631:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913193216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075874.289638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075874.289641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075874.289643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075874.289646:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1579 00000100:00080000:1.0:1769075874.289650:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075874.289654:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075874.289658:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075874.289671:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913193344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075874.289687:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769075874.289690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075874.289696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075874.289700:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075874.289706:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1579 00000100:00080000:1.0:1769075874.289710:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075874.289715:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075874.289722:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075874.289737:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913193472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075874.289779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075874/real 1769075874] req@ffff9350062a5880 x1855008913192704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075874.289809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913192704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075874.289825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075874.289831:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075874.289836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913192704:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075874.289849:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075874.289883:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075874.289898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075874/real 1769075874] req@ffff9350062a5500 x1855008913192832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075874.289913:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913192832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075874.289925:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075874.289929:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075874.289935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075874.289968:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075874.289990:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075874.290103:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075874/real 1769075874] req@ffff9350062a4380 x1855008913192960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075874.290134:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913192960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075874.290152:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075874.290159:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075874.290167:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913192960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075874.290181:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913193088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075874.290220:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075874.290237:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075874/real 1769075874] req@ffff9350062a5f80 x1855008913193088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075874.290253:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913193088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075874.290264:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075874.290268:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075874.290273:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913193088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075874.290291:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913193216:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075874.290312:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075874.290325:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075874/real 1769075874] req@ffff9350062a4a80 x1855008913193216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075874.290339:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913193216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075874.290349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075874.290353:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075874.290357:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913193216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075874.290374:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913193344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075874.290393:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075874.290404:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075874/real 1769075874] req@ffff9350062a5180 x1855008913193344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075874.290419:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913193344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075874.290438:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075874.290441:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075874.290445:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913193344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075874.290451:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913193472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075874.290469:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075874.290478:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075874/real 1769075874] req@ffff9350062a5c00 x1855008913193472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075874.290491:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913193472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075874.290502:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075874.290506:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075874.290510:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913193472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075874.993250:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075874.993299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075876.017227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075876.017468:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075877.041222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075877.041376:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075878.065146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075878.065252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075879.089168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075879.089404:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075879.409286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075879.409302:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075879.409308:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075879.409323:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1585 00000100:00080000:3.0:1769075879.409339:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075879.409348:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075879.409409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913193600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075879.409467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075879.409475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075879.409480:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075879.409489:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1585 00000100:00080000:3.0:1769075879.409498:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075879.409506:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075879.409526:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075879.409558:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913193728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075879.409585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075879.409590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075879.409593:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075879.409599:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1585 00000100:00080000:3.0:1769075879.409606:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075879.409613:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075879.409621:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075879.409637:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913193856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075879.409650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075879.409655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075879.409658:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075879.409663:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1585 00000100:00080000:3.0:1769075879.409680:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075879.409685:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075879.409694:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075879.409727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913193984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075879.409750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075879.409756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075879.409760:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075879.409771:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1585 00000100:00080000:3.0:1769075879.409781:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075879.409793:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075879.409807:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075879.409845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913194112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075879.409868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075879.409886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075879.409893:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075879.409907:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1585 00000100:00080000:3.0:1769075879.409944:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075879.409955:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075879.409974:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075879.409998:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913194240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075879.410013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075879.410017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075879.410020:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075879.410025:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1585 00000100:00080000:3.0:1769075879.410033:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075879.410044:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075879.410103:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075879.410120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913194368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075880.113146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075880.113258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075881.137223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075881.137452:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075882.161239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075882.161332:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075883.185205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075883.185234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075884.209190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075884.209201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075884.209292:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075884.529302:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913194496:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769075884.529395:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075884.529452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075884/real 1769075884] req@ffff935002a62680 x1855008913194496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075884.529493:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913194496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075884.529504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075884.529508:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075884.529511:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913194496:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075884.529533:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913194624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075884.529552:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075884.529561:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075884/real 1769075884] req@ffff935002a61880 x1855008913194624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075884.529570:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913194624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075884.529576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075884.529578:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075884.529581:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913194624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075884.529585:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913194752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075884.529596:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075884.529602:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075884/real 1769075884] req@ffff935002a62a00 x1855008913194752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075884.529609:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913194752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075884.529614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075884.529616:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075884.529618:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913194752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075884.529621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913194880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075884.529632:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075884.529641:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075884/real 1769075884] req@ffff935002a60000 x1855008913194880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075884.529653:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913194880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075884.529662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075884.529665:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075884.529669:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913194880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075884.529675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195008:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075884.529694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075884.529704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075884/real 1769075884] req@ffff935002a61c00 x1855008913195008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075884.529713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913195008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075884.529719:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075884.529721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075884.529723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075884.529727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075884.529738:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075884.529744:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075884/real 1769075884] req@ffff935002a62300 x1855008913195136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075884.529752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913195136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075884.529757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075884.529758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075884.529760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075884.529764:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075884.529773:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075884.529779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075884/real 1769075884] req@ffff935002a61500 x1855008913195264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075884.529790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913195264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075884.529798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075884.529802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075884.529805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769075885.233175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075885.233175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075886.257199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075886.257199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075887.281158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075887.281164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075888.305193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075888.305193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075889.329129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075889.329179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:1.0:1769075889.649236:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075889.649250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075889.649256:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075889.649269:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1595 00000100:00080000:1.0:1769075889.649282:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075889.649290:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075889.649345:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913195392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075889.649387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075889.649391:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075889.649393:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075889.649397:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1595 00000100:00080000:1.0:1769075889.649401:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075889.649404:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075889.649415:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075889.649429:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913195520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075889.649439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075889.649454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075889.649457:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075889.649461:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1595 00000100:00080000:1.0:1769075889.649463:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075889.649467:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075889.649471:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075889.649485:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913195392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769075889.649486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913195648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075889.649507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075889.649510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075889.649512:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075889.649516:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1595 00000100:00080000:1.0:1769075889.649521:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075889.649524:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075889.649528:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075889.649549:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913195776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075889.649569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075889.649573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075889.649574:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075889.649579:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1595 00000100:00080000:1.0:1769075889.649583:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075889.649586:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075889.649591:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075889.649600:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769075889.649607:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913195904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075889.649628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075889.649632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075889.649634:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075889.649638:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1595 00000100:00080000:1.0:1769075889.649659:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075889.649664:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075889.649669:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075889.649682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913196032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075889.649698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075889.649702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075889/real 1769075889] req@ffff9350062a5c00 x1855008913195392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075889.649704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075889.649727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075889.649741:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1595 00000100:00080000:1.0:1769075889.649801:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075889.649805:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075889.649810:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075889.649828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5880 x1855008913196160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075889.649832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5c00 x1855008913195392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075889.649866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075889.649873:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075889.649878:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913195392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769075889.649886:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913195520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075889.649915:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075889.649928:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075889/real 1769075889] req@ffff9350062a5180 x1855008913195520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075889.649948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5180 x1855008913195520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075889.649958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075889.649961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075889.649965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913195520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075889.649993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913195648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075889.650010:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075889.650019:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075889/real 1769075889] req@ffff9350062a4a80 x1855008913195648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075889.650031:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4a80 x1855008913195648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075889.650039:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075889.650042:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075889.650045:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913195648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075889.650093:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913195776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075889.650122:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075889.650139:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075889/real 1769075889] req@ffff9350062a5f80 x1855008913195776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075889.650158:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5f80 x1855008913195776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075889.650193:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075889.650199:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075889.650204:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913195776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075889.650213:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195904:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769075889.650260:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075889.650275:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075889/real 1769075889] req@ffff9350062a4380 x1855008913195904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075889.650307:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a4380 x1855008913195904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075889.650315:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075889.650319:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075889.650322:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913195904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769075889.650327:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913196032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075889.650343:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075889.650367:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075889/real 1769075889] req@ffff9350062a5500 x1855008913196032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075889.650378:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5500 x1855008913196032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075889.650387:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075889.650392:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075889.650397:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913196032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075889.650406:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913196160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769075889.650434:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769075889.650457:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075889/real 1769075889] req@ffff9350062a5880 x1855008913196160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075889.650477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350062a5880 x1855008913196160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075889.650487:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075889.650490:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769075889.650493:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350062a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913196160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769075890.353220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075890.353258:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075891.377191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075891.377204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075891.377246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075892.401191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075892.401192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075892.401229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075893.425197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075893.425197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075894.449269:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075894.449269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075894.769282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075894.769293:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075894.769297:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075894.769306:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1600 00000100:00080000:3.0:1769075894.769319:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075894.769326:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075894.769388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913196288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075894.769418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075894.769422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075894.769424:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075894.769428:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1600 00000100:00080000:3.0:1769075894.769431:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075894.769434:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075894.769465:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075894.769479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913196416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075894.769491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075894.769495:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075894.769499:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075894.769508:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1600 00000100:00080000:3.0:1769075894.769537:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075894.769543:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075894.769549:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075894.769567:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913196544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075894.769577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075894.769581:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075894.769584:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075894.769588:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1600 00000100:00080000:3.0:1769075894.769609:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075894.769613:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075894.769621:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075894.769693:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913196672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075894.769707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075894.769726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075894.769729:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075894.769734:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1600 00000100:00080000:3.0:1769075894.769738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075894.769749:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075894.769765:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075894.769807:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913196800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075894.769871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075894.769879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075894.769891:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075894.769903:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1600 00000100:00080000:3.0:1769075894.769916:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075894.769922:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075894.769931:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075894.769979:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913196928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075894.769994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075894.770004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075894.770010:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075894.770032:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1600 00000100:00080000:3.0:1769075894.770040:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075894.770044:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075894.770049:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075894.770147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913197056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075895.473148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075895.473148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075895.473209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769075895.801347:0:25145:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff935011534b08),name=/ 00000002:00010000:1.0:1769075895.801374:0:25145:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769075895.801390:0:25145:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9350066d5000 type 13 mode 30 res 8589934599/1 (0 0) 00010000:00010000:1.0:1769075895.801445:0:25145:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e33d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 25145 initiator: MDT0 00010000:00010000:1.0:1769075895.801454:0:25145:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e33d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 25145 initiator: MDT0 00010000:00010000:1.0:1769075895.801479:0:25145:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9350067ee000 lock: ffff935004aa1000/0x410246667097e33d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 25145 initiator: MDT0 00000100:00080000:1.0:1769075895.801502:0:25145:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9350062a5880 x1855008913197184/t0(0) o101->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 576/1152 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'chmod.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1769075895.801512:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075896.497179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075896.497200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075896.497250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075896.817206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075897.521152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075897.521152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075897.521230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075897.841237:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075898.545222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075898.545254:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075898.865146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075899.569133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075899.569166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769075899.889162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075899.889169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075899.889172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075899.889181:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.889193:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1605 00000100:00080000:3.0:1769075899.889204:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.889212:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075899.889252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913197312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.889282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.889285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075899.889286:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.889289:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1605 00000100:00080000:3.0:1769075899.889292:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.889295:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075899.889304:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075899.889313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913197440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.889320:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.889322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075899.889323:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.889326:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1605 00000100:00080000:3.0:1769075899.889328:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.889330:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075899.889333:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075899.889338:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913197568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.889345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.889347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075899.889348:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.889351:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1605 00000100:00080000:3.0:1769075899.889353:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.889356:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075899.889358:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075899.889365:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913197696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.889372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.889376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075899.889378:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.889385:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1605 00000100:00080000:3.0:1769075899.889388:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.889390:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075899.889393:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075899.889404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913197824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.889410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.889412:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075899.889414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.889416:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1605 00000100:00080000:3.0:1769075899.889419:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.889421:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075899.889424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075899.889431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913197952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.889437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.889440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075899.889441:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.889443:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1605 00000100:00080000:3.0:1769075899.889445:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.889447:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075899.889451:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075899.889461:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913198080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075900.593266:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075900.593271:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075900.913244:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075901.617195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075901.617204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075901.937184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075902.641170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075902.641171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075902.961249:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075903.665151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075903.665187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075903.985257:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075904.689115:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075904.689137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769075905.009173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075905.009183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075905.009188:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075905.009196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075905.009197:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1610 00000100:00080000:3.0:1769075905.009211:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075905.009220:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075905.009257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913198208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075905.009283:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075905.009286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075905.009288:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075905.009291:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1610 00000100:00080000:3.0:1769075905.009294:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075905.009296:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075905.009305:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075905.009315:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913198336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075905.009323:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075905.009326:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075905.009327:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075905.009332:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1610 00000100:00080000:3.0:1769075905.009334:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075905.009340:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075905.009344:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075905.009353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913198464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075905.009358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075905.009361:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075905.009362:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075905.009365:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1610 00000100:00080000:3.0:1769075905.009367:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075905.009369:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075905.009374:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075905.009385:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913198592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075905.009395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075905.009398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075905.009401:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075905.009405:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1610 00000100:00080000:3.0:1769075905.009410:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075905.009417:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075905.009424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075905.009439:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913198720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075905.009446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075905.009449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075905.009451:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075905.009454:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1610 00000100:00080000:3.0:1769075905.009458:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075905.009464:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075905.009471:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075905.009483:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913198848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075905.009512:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075905.009517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075905.009520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075905.009529:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1610 00000100:00080000:3.0:1769075905.009536:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075905.009543:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075905.009549:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075905.009558:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913198976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075905.713129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075905.713139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075906.033203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075906.737235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075906.737249:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075907.057195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075907.761167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075907.761169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075907.761234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075908.081158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075908.785183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075908.785183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075909.105180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075909.809170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075909.809179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075910.129185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075910.129230:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075910.129242:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075910.129247:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075910.129260:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1615 00000100:00080000:3.0:1769075910.129273:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075910.129281:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075910.129349:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913199104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075910.129384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075910.129388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075910.129391:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075910.129395:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1615 00000100:00080000:3.0:1769075910.129419:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075910.129424:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075910.129452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075910.129465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913199232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075910.129476:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075910.129480:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075910.129482:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075910.129490:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1615 00000100:00080000:3.0:1769075910.129517:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075910.129523:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075910.129529:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075910.129544:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913199360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075910.129555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075910.129559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075910.129561:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075910.129564:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1615 00000100:00080000:3.0:1769075910.129587:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075910.129591:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075910.129595:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075910.129611:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913199488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075910.129622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075910.129627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075910.129629:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075910.129634:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1615 00000100:00080000:3.0:1769075910.129641:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075910.129645:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075910.129651:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075910.129664:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913199616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075910.129674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075910.129678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075910.129682:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075910.129688:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1615 00000100:00080000:3.0:1769075910.129697:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075910.129705:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075910.129715:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075910.129750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913199744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075910.129763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075910.129772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075910.129773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075910.129778:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1615 00000100:00080000:3.0:1769075910.129807:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075910.129818:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075910.129829:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075910.129851:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913199872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075910.833154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075910.833157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075911.153214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075911.857150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075911.857164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075912.177152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075912.881143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075912.881178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075913.201189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075913.905163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075913.905196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075913.905232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075914.225154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075914.929197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075914.929216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075915.249176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075915.953147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075915.953168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075916.273137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075916.977198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075916.977198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075917.297153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075918.001219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075918.001221:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075918.321119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075919.025135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075919.025171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075919.025184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075919.345174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075920.049143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075920.049144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075920.369154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075920.369162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075920.369169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075920.369173:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075920.369182:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1626 00000100:00080000:3.0:1769075920.369194:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075920.369203:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075920.369293:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913200896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075920.369335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075920.369339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075920.369342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075920.369347:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1626 00000100:00080000:3.0:1769075920.369352:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075920.369356:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075920.369366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075920.369381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913201024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075920.369407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075920.369411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075920.369414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075920.369429:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1626 00000100:00080000:3.0:1769075920.369434:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075920.369439:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075920.369445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075920.369460:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913201152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075920.369471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075920.369475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075920.369477:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075920.369483:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1626 00000100:00080000:3.0:1769075920.369487:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075920.369492:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075920.369499:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075920.369516:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913201280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075920.369533:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075920.369537:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075920.369540:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075920.369548:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1626 00000100:00080000:3.0:1769075920.369559:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075920.369566:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075920.369573:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075920.369590:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913201408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075920.369610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075920.369615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075920.369617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075920.369622:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1626 00000100:00080000:3.0:1769075920.369637:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075920.369643:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075920.369649:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075920.369669:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913201536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075920.369683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075920.369687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075920.369690:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075920.369695:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1626 00000100:00080000:3.0:1769075920.369702:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075920.369709:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075920.369714:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075920.369727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913201664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075921.073143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075921.073143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075921.393307:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075922.097180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075922.097186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769075922.097272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075922.097280:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075922.417258:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075923.121159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075923.121300:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075923.441416:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075924.145149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075924.145224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075924.145258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075924.465355:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075925.169204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075925.169351:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075925.489286:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075926.193115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075926.193223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075926.513372:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075927.217163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075927.217220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075927.217280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075927.537164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075928.241216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075928.241248:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075928.241270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075928.561237:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075929.265184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075929.265184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075929.265219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075929.585189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075930.289146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075930.289207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075930.289267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075930.609151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075930.609164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075930.609174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075930.609178:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075930.609187:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1636 00000100:00080000:3.0:1769075930.609198:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075930.609204:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075930.609239:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913202688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075930.609262:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075930.609265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075930.609267:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075930.609270:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1636 00000100:00080000:3.0:1769075930.609273:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075930.609275:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075930.609285:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075930.609308:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913202816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075930.609317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075930.609323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075930.609330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075930.609340:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1636 00000100:00080000:3.0:1769075930.609342:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075930.609344:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075930.609348:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075930.609360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913202944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075930.609367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075930.609369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075930.609371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075930.609373:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1636 00000100:00080000:3.0:1769075930.609375:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075930.609378:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075930.609381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075930.609388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913203072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075930.609396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075930.609399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075930.609400:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075930.609403:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1636 00000100:00080000:3.0:1769075930.609420:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075930.609423:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075930.609428:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075930.609443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913203200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075930.609452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075930.609456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075930.609458:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075930.609472:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1636 00000100:00080000:3.0:1769075930.609475:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075930.609478:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075930.609483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075930.609494:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913203328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075930.609509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075930.609526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075930.609534:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075930.609548:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1636 00000100:00080000:3.0:1769075930.609554:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075930.609557:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075930.609563:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075930.609582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913203456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075931.313190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075931.313234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075931.313304:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075931.633155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075932.337189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075932.337199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075932.657161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075933.361165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075933.361178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075933.681186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075934.385152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075934.385152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075934.705169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075935.409207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075935.409229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075935.729155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075935.729164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075935.729172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075935.729178:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075935.729190:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1641 00000100:00080000:3.0:1769075935.729203:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075935.729223:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075935.729270:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913203584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075935.729311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075935.729317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075935.729320:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075935.729327:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1641 00000100:00080000:3.0:1769075935.729333:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075935.729340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075935.729356:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075935.729393:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913203712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075935.729407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075935.729410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075935.729413:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075935.729418:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1641 00000100:00080000:3.0:1769075935.729441:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075935.729455:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075935.729466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075935.729481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913203840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075935.729491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075935.729495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075935.729497:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075935.729503:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1641 00000100:00080000:3.0:1769075935.729507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075935.729516:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075935.729526:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075935.729553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913203968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075935.729584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075935.729592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075935.729599:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075935.729610:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1641 00000100:00080000:3.0:1769075935.729617:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075935.729621:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075935.729625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075935.729638:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913204096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075935.729664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075935.729668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075935.729670:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075935.729675:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1641 00000100:00080000:3.0:1769075935.729679:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075935.729688:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075935.729696:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075935.729711:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913204224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075935.729736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075935.729739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075935.729741:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075935.729744:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1641 00000100:00080000:3.0:1769075935.729747:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075935.729751:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075935.729759:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075935.729774:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913204352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075936.433163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075936.433166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075936.753163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075937.457240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075937.457241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075937.777128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075938.481133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075938.481135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075938.801121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075939.505208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075939.505208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075939.825259:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075940.529138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075940.529153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075940.529212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075940.849172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075941.553127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075941.553165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075941.553172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075941.873171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075942.577150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075942.577159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075942.577224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075942.897139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075943.601134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075943.601134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075943.601211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075943.921145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075944.625210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075944.625211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075944.625262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075944.945130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075945.649197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075945.649217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075945.649284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075945.969190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075946.673169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075946.673169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075946.673312:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075946.993181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075947.697137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075947.697162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075947.697248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075948.017211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075948.721205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075948.721253:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075949.041169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075949.745122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075949.745127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075950.065163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075950.769189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075950.769189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769075951.089262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075951.089292:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075951.089299:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075951.089313:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1656 00000100:00080000:1.0:1769075951.089331:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075951.089344:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075951.089377:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075951.089405:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5500 x1855008913206272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075951.089445:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075951.089451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075951.089454:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075951.089459:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1656 00000100:00080000:1.0:1769075951.089464:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075951.089468:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075951.089480:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075951.089495:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4380 x1855008913206400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075951.089508:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075951.089512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075951.089514:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075951.089519:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1656 00000100:00080000:1.0:1769075951.089522:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075951.089527:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075951.089533:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075951.089546:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5f80 x1855008913206528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075951.089558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075951.089578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075951.089581:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075951.089587:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1656 00000100:00080000:1.0:1769075951.089592:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075951.089595:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075951.089599:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075951.089610:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a4a80 x1855008913206656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075951.089624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075951.089656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075951.089660:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075951.089666:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1656 00000100:00080000:1.0:1769075951.089670:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075951.089676:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075951.089683:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075951.089701:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5180 x1855008913206784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075951.089743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075951.089748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075951.089751:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075951.089756:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1656 00000100:00080000:1.0:1769075951.089761:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075951.089766:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075951.089774:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075951.089794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350062a5c00 x1855008913206912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075951.089807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075951.089812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075951.089814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075951.089818:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1656 00000100:00080000:1.0:1769075951.089825:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075951.089848:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075951.089858:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075951.089890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913207040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075951.793172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075951.793180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075951.793278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075952.113208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075952.817126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075952.817126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075952.817366:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075953.137156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075953.841200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075953.841212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075953.841282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075954.161211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075954.865219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075954.865223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075954.865284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075955.185176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075955.889288:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075955.889301:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075956.209217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075956.913197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075956.913252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075957.233185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075957.937122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075957.937178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075958.257179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075958.961172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075958.961335:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075959.281228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075959.985139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075959.985187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075959.985205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075960.305119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075961.009177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075961.009222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075961.009282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075961.329273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075961.329288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075961.329295:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075961.329303:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769075961.329316:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1667 00000100:00080000:1.0:1769075961.329335:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075961.329346:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769075961.329431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913208064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075961.329488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075961.329496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075961.329501:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075961.329510:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1667 00000100:00080000:1.0:1769075961.329519:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075961.329527:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075961.329569:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075961.329603:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913208192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075961.329652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075961.329658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075961.329663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075961.329672:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1667 00000100:00080000:1.0:1769075961.329678:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075961.329683:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075961.329692:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075961.329712:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913208320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075961.329725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075961.329730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075961.329733:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075961.329738:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1667 00000100:00080000:1.0:1769075961.329742:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075961.329756:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075961.329764:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769075961.329783:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913208448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075961.329796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075961.329802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075961.329805:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075961.329811:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1667 00000100:00080000:1.0:1769075961.329817:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075961.329825:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075961.329831:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075961.329850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913208576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075961.329908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075961.329941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075961.329945:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075961.329956:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1667 00000100:00080000:1.0:1769075961.329961:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075961.329973:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075961.329981:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075961.330015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913208704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075961.330118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075961.330137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075961.330144:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075961.330157:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1667 00000100:00080000:1.0:1769075961.330171:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075961.330188:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769075961.330197:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075961.330214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913208832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075962.033191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075962.033269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075962.353291:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075963.057286:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075963.057326:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075963.377334:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075964.081209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075964.081267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075964.401181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075965.105183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075965.105195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000008:00020000:3.0:1769075965.425252:0:14430:0:(osc_cache.c:945:osc_extent_wait()) extent ffff9350105269c0@{[0 -> 255/255], [3|0|+|rpc|wiuY|ffff93501005ae00], [1703936|10|+|-|ffff93500316b600|256|ffff93500134a7c0]} lustre-OST0001-osc-ffff9350067ee000: wait ext to 0 timedout, recovery in progress? 00000008:00020000:3.0:1769075965.438435:0:14430:0:(osc_cache.c:945:osc_extent_wait()) ### extent: 000000009c567a31 ns: lustre-OST0001-osc-ffff9350067ee000 lock: ffff93500316b600/0x410246667097e2aa lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800029400020000 nid: local remote: 0x48bf02eaaaf97eb4 expref: -99 pid: 24287 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769075966.129206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075966.129289:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075966.449270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075966.449285:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075966.449292:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075966.449306:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1672 00000100:00080000:3.0:1769075966.449323:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075966.449333:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075966.449417:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913208960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075966.449494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075966.449501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075966.449505:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075966.449512:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1672 00000100:00080000:3.0:1769075966.449518:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075966.449524:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075966.449538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075966.449561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913209088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075966.449589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075966.449593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075966.449596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075966.449603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1672 00000100:00080000:3.0:1769075966.449611:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075966.449615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075966.449621:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075966.449636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913209216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075966.449652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075966.449656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075966.449658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075966.449662:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1672 00000100:00080000:3.0:1769075966.449669:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075966.449673:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075966.449678:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075966.449695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913209344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075966.449714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075966.449720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075966.449723:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075966.449730:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1672 00000100:00080000:3.0:1769075966.449735:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075966.449743:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075966.449750:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075966.449772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913209472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075966.449821:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075966.449825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075966.449834:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075966.449853:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1672 00000100:00080000:3.0:1769075966.449861:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075966.449868:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075966.449874:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075966.449900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913209600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075966.449950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075966.449971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075966.449975:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075966.449984:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1672 00000100:00080000:3.0:1769075966.449989:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075966.449994:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075966.450003:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075966.450022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913209728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075967.153222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075967.153240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075967.153315:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075967.473316:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075968.177257:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075968.177257:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075968.497175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075969.201239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075969.201251:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075969.521256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075970.225188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075970.225209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075970.545240:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075971.249222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075971.249267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075971.569215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075971.569224:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075971.569231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075971.569241:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075971.569254:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1677 00000100:00080000:3.0:1769075971.569272:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075971.569283:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075971.569338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913209856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075971.569418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075971.569425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075971.569428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075971.569434:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1677 00000100:00080000:3.0:1769075971.569439:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075971.569445:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075971.569459:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075971.569483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913209984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075971.569499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075971.569505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075971.569508:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075971.569514:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1677 00000100:00080000:3.0:1769075971.569520:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075971.569525:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075971.569539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075971.569574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913210112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075971.569596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075971.569602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075971.569605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075971.569611:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1677 00000100:00080000:3.0:1769075971.569616:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075971.569623:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075971.569634:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075971.569668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913210240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075971.569680:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075971.569692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075971.569698:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075971.569708:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1677 00000100:00080000:3.0:1769075971.569749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075971.569759:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075971.569770:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075971.569806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913210368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075971.569921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075971.569941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075971.569951:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075971.569967:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1677 00000100:00080000:3.0:1769075971.569987:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075971.570005:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075971.570015:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075971.570042:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913210496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075971.570127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075971.570138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075971.570149:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075971.570178:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1677 00000100:00080000:3.0:1769075971.570196:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075971.570222:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075971.570241:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075971.570270:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913210624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075972.273186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075972.273237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075973.297186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075973.297230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075974.321141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075974.321207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075975.345219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075975.345234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075976.369218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075976.369255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075977.393173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075977.393252:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075977.713352:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075978.417182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075978.417183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075978.737231:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075979.441192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075979.441343:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075979.761223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075980.465162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075980.465268:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075980.785200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075981.489250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075981.489323:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075981.809184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075982.513171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075982.513232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075982.513270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075982.833268:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075983.537204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075983.537204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075983.857206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075984.561179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075984.561179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075984.561249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075984.881172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075985.585253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075985.585266:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075985.905183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075986.609165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075986.609182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075986.929287:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075987.633200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075987.633226:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075987.953296:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075988.657210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075988.657218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075988.977286:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075989.681273:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075989.681290:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075990.001579:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075990.705189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075990.705189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075991.025244:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075991.729217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075991.729239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075992.049290:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769075992.753219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075992.753226:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075993.073370:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075993.777234:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075993.777235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075994.097192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075994.801205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075994.801261:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075995.121159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075995.825185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075995.825282:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769075996.145190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075996.849178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075996.849267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075997.169187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769075997.169223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075997.169235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075997.169242:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075997.169257:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1702 00000100:00080000:3.0:1769075997.169277:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075997.169290:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769075997.169339:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913214336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075997.169380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075997.169385:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075997.169388:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075997.169393:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1702 00000100:00080000:3.0:1769075997.169398:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075997.169402:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075997.169416:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075997.169433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913214464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075997.169450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075997.169454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075997.169457:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075997.169468:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1702 00000100:00080000:3.0:1769075997.169474:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075997.169485:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075997.169492:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075997.169510:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913214592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075997.169523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075997.169528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075997.169531:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075997.169539:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1702 00000100:00080000:3.0:1769075997.169554:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075997.169560:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075997.169575:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769075997.169596:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913214720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075997.169609:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075997.169614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075997.169618:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075997.169625:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1702 00000100:00080000:3.0:1769075997.169635:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075997.169652:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075997.169671:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075997.169695:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913214848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075997.169737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075997.169747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075997.169756:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075997.169763:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1702 00000100:00080000:3.0:1769075997.169768:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075997.169773:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075997.169782:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075997.169806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913214976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075997.169832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075997.169839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075997.169842:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075997.169857:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1702 00000100:00080000:3.0:1769075997.169867:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075997.169882:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769075997.169892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075997.169914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913215104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075997.873197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769075997.873418:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075998.193212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075998.897147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075998.897219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075998.897250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075999.217162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769075999.921131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769075999.921143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076000.241116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076000.945169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076000.945188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076000.945233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076001.265174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076001.969220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076001.969252:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076001.969364:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076002.289200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076002.993179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076002.993180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076002.993259:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076003.313172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076004.017197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076004.017227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076004.017299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076004.337161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076005.041166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076005.041175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076005.041214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076005.361157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076006.065183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076006.065184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076006.065222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076006.385225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076007.089256:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076007.089280:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076007.089350:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076007.409208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076008.113219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076008.113219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076008.113279:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076008.433138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076009.137224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076009.137225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076009.137277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076009.457181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076010.161242:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076010.161249:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076010.161301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076011.185212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076011.185431:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076012.209215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076012.209280:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769076012.529245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076012.529256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076012.529262:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076012.529294:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1718 00000100:00080000:1.0:1769076012.529309:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076012.529319:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076012.529378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913217024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076012.529428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076012.529434:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076012.529437:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076012.529442:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1718 00000100:00080000:1.0:1769076012.529447:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076012.529452:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076012.529468:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076012.529486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913217152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076012.529532:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076012.529540:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076012.529544:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076012.529555:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1718 00000100:00080000:1.0:1769076012.529562:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076012.529570:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076012.529581:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076012.529606:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913217280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076012.529641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076012.529646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076012.529649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076012.529657:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1718 00000100:00080000:1.0:1769076012.529666:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076012.529671:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076012.529679:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076012.529751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913217408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076012.529782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076012.529795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076012.529800:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076012.529808:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1718 00000100:00080000:1.0:1769076012.529816:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076012.529823:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076012.529834:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076012.529860:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913217536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076012.529891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076012.529899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076012.529907:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076012.529917:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1718 00000100:00080000:1.0:1769076012.529922:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076012.529926:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076012.529931:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076012.529969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913217664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076012.530009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076012.530016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076012.530024:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076012.530043:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1718 00000100:00080000:1.0:1769076012.530053:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076012.530071:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076012.530165:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076012.530198:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913217792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076013.233151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076013.233166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076013.553201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076014.257152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076014.257201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076014.577169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076015.281282:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076015.281292:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076015.601170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076016.305223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076016.305246:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076016.625185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076017.329182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076017.329182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076017.649231:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076017.649254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076017.649265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076017.649272:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076017.649288:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1723 00000100:00080000:3.0:1769076017.649305:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076017.649315:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076017.649373:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913217920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076017.649416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076017.649422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076017.649438:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076017.649445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1723 00000100:00080000:3.0:1769076017.649451:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076017.649457:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076017.649474:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076017.649498:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913218048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076017.649541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076017.649550:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076017.649555:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076017.649567:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1723 00000100:00080000:3.0:1769076017.649655:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076017.649675:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076017.649692:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076017.649725:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913218176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076017.649742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076017.649750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076017.649756:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076017.649772:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1723 00000100:00080000:3.0:1769076017.649782:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076017.649796:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076017.649811:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076017.649836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913218304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076017.649882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076017.649896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076017.649901:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076017.649911:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1723 00000100:00080000:3.0:1769076017.649919:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076017.649931:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076017.649943:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076017.649976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913218432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076017.649995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076017.650008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076017.650011:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076017.650018:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1723 00000100:00080000:3.0:1769076017.650024:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076017.650030:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076017.650037:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076017.650116:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913218560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076017.650134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076017.650148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076017.650156:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076017.650170:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1723 00000100:00080000:3.0:1769076017.650177:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076017.650182:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076017.650191:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076017.650211:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913218688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076018.353175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076018.353215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076018.673177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076019.377146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076019.377151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076019.697263:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076020.401153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076020.401153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076020.401225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076020.721329:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076021.425253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076021.425352:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076021.425437:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076021.745232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076022.449216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076022.449216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076022.449394:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076022.769197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076023.473194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076023.473194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076023.473285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076023.793191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076024.497132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076024.497133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076024.497198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076024.817196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076025.521247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076025.521249:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076025.521380:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076025.841209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076026.545195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076026.545227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076026.545400:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076026.865276:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076027.569189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076027.569267:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076027.569311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076027.889243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076028.593222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076028.593285:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076028.913314:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076029.617212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076029.617240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076029.937186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076030.641170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076030.641185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076030.961151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076031.665258:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076031.665328:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076031.985174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076032.689188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076032.689260:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076032.689265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076033.009182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076033.009198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076033.009206:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076033.009223:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1738 00000100:00080000:1.0:1769076033.009239:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076033.009456:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076033.009525:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913220608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076033.009578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076033.009586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076033.009590:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076033.009597:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1738 00000100:00080000:1.0:1769076033.009611:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076033.009617:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076033.009632:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076033.009649:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913220736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076033.009676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076033.009684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076033.009687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076033.009694:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1738 00000100:00080000:1.0:1769076033.009698:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076033.009707:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076033.009716:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076033.009732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913220864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076033.009745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076033.009750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076033.009754:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076033.009759:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1738 00000100:00080000:1.0:1769076033.009764:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076033.009769:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076033.009775:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076033.009788:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913220992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076033.009810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076033.009816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076033.009819:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076033.009826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1738 00000100:00080000:1.0:1769076033.009833:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076033.009839:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076033.009846:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076033.009871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913221120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076033.009887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076033.009893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076033.009897:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076033.009912:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1738 00000100:00080000:1.0:1769076033.009927:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076033.009948:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076033.009962:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076033.009987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913221248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076033.010150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076033.010168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076033.010181:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076033.010196:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1738 00000100:00080000:1.0:1769076033.010205:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076033.010218:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076033.010239:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076033.010274:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913221376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076033.713263:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076033.713282:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076034.737198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076034.737198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076035.761190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076035.761190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076035.761287:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076036.785181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076036.785187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076036.785405:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076037.809187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076037.809281:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076037.809361:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076038.129302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076038.129313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076038.129319:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076038.129331:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1743 00000100:00080000:3.0:1769076038.129347:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076038.129357:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076038.129411:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913221504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076038.129451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076038.129457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076038.129459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076038.129465:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1743 00000100:00080000:3.0:1769076038.129469:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076038.129474:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076038.129487:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076038.129502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913221632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076038.129515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076038.129520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076038.129522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076038.129527:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1743 00000100:00080000:3.0:1769076038.129531:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076038.129536:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076038.129542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076038.129558:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913221760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076038.129571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076038.129576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076038.129578:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076038.129585:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1743 00000100:00080000:3.0:1769076038.129590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076038.129594:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076038.129601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076038.129618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913221888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076038.129658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076038.129663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076038.129666:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076038.129676:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1743 00000100:00080000:3.0:1769076038.129682:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076038.129688:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076038.129697:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076038.129719:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913222016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076038.129732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076038.129737:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076038.129740:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076038.129758:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1743 00000100:00080000:3.0:1769076038.129772:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076038.129782:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076038.129792:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076038.129810:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913222144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076038.129833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076038.129838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076038.129849:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076038.129878:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1743 00000100:00080000:3.0:1769076038.129904:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076038.129922:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076038.129957:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076038.129992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913222272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076038.833197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076038.833401:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076039.857177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076039.857316:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076039.857329:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076040.881240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076040.881276:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076040.881405:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076041.905172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076041.905242:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076041.905391:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076042.929194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076042.929204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076042.929453:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076043.249210:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076043.249222:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076043.249227:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076043.249239:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1748 00000100:00080000:3.0:1769076043.249253:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076043.249282:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076043.249347:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913222400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076043.249388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076043.249393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076043.249396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076043.249402:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1748 00000100:00080000:3.0:1769076043.249406:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076043.249412:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076043.249431:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076043.249460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913222528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076043.249498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076043.249505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076043.249509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076043.249520:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1748 00000100:00080000:3.0:1769076043.249528:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076043.249534:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076043.249542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076043.249561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913222656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076043.249587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076043.249593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076043.249596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076043.249603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1748 00000100:00080000:3.0:1769076043.249609:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076043.249613:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076043.249619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076043.249634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913222784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076043.249649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076043.249662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076043.249666:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076043.249682:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1748 00000100:00080000:3.0:1769076043.249688:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076043.249694:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076043.249700:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076043.249720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913222912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076043.249753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076043.249772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076043.249777:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076043.249786:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1748 00000100:00080000:3.0:1769076043.249794:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076043.249806:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076043.249822:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076043.249855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913223040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076043.249892:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076043.249899:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076043.249903:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076043.249912:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1748 00000100:00080000:3.0:1769076043.249923:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076043.249933:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076043.249948:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076043.249967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913223168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076043.953216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076043.953279:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076043.953316:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076043.953397:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076044.977213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076044.977243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076044.977363:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076046.001251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076046.001301:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076047.025205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076047.025232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076048.049241:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076048.049262:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769076048.369276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076048.369288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076048.369294:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076048.369308:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1754 00000100:00080000:3.0:1769076048.369337:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076048.369347:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076048.369398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913223296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076048.369435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076048.369440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076048.369443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076048.369449:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1754 00000100:00080000:3.0:1769076048.369453:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076048.369459:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076048.369472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076048.369487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913223424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076048.369501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076048.369506:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076048.369509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076048.369513:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1754 00000100:00080000:3.0:1769076048.369517:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076048.369522:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076048.369530:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076048.369547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913223552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076048.369570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076048.369576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076048.369579:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076048.369585:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1754 00000100:00080000:3.0:1769076048.369590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076048.369596:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076048.369603:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076048.369618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913223680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076048.369632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076048.369637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076048.369640:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076048.369646:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1754 00000100:00080000:3.0:1769076048.369651:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076048.369655:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076048.369661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076048.369687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913223808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076048.369701:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076048.369705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076048.369707:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076048.369712:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1754 00000100:00080000:3.0:1769076048.369717:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076048.369721:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076048.369730:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076048.369750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913223936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076048.369784:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076048.369806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076048.369818:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076048.369825:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1754 00000100:00080000:3.0:1769076048.369836:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076048.369855:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076048.369876:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076048.369902:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913224064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076049.073187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076049.073188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076050.097227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076050.097227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076050.097315:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076051.121183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076051.121183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076051.121416:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076052.145231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076052.145245:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076052.145358:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076053.169207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076053.169231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076053.169299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076053.489248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076053.489268:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076053.489276:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076053.489291:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1759 00000100:00080000:3.0:1769076053.489309:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076053.489319:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076053.489396:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913224192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076053.489452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076053.489458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076053.489460:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076053.489466:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1759 00000100:00080000:3.0:1769076053.489471:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076053.489476:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076053.489491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076053.489526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913224320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076053.489538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076053.489544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076053.489548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076053.489564:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1759 00000100:00080000:3.0:1769076053.489568:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076053.489587:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076053.489597:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076053.489615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913224448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076053.489630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076053.489636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076053.489639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076053.489645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1759 00000100:00080000:3.0:1769076053.489673:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076053.489679:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076053.489686:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076053.489700:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913224576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076053.489712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076053.489717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076053.489720:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076053.489730:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1759 00000100:00080000:3.0:1769076053.489738:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076053.489745:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076053.489757:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076053.489783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913224704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076053.489801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076053.489807:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076053.489812:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076053.489838:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1759 00000100:00080000:3.0:1769076053.489851:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076053.489868:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076053.489884:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076053.489923:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913224832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076053.489961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076053.489965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076053.489968:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076053.489974:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1759 00000100:00080000:3.0:1769076053.489980:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076053.489984:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076053.489990:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076053.490019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913224960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076053.681249:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076054.193199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076054.193202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076054.193257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076054.705155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076055.217164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076055.217190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076055.217301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076055.729165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076056.241175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076056.241188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076056.241241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076056.561498:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769076056.561557:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076056.561592:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076056.561673:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076056.561693:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076056.561711:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076056.561730:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076056.561750:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076056.561766:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769076056.753152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076057.265164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076057.265177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076057.265262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076057.777170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076058.289165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076058.289189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076058.289292:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076058.609304:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076058.801140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076059.313211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076059.313223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076059.313510:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076059.633341:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076059.825144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076060.337198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076060.337223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076060.657171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076060.849184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076061.361177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076061.361178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076061.361230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076061.681204:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076061.873201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076062.385218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076062.385218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076062.385262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076062.705174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076062.897197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076063.409148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076063.409155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076063.409216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076063.729116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076063.729175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076063.729181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076063.729185:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076063.729194:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1769 00000100:00080000:1.0:1769076063.729211:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076063.729218:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076063.729254:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913225984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076063.729278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076063.729282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076063.729283:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076063.729287:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1769 00000100:00080000:1.0:1769076063.729289:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076063.729292:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076063.729301:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076063.729310:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913226112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076063.729317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076063.729319:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076063.729321:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076063.729324:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1769 00000100:00080000:1.0:1769076063.729326:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076063.729328:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076063.729332:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076063.729339:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913226240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076063.729345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076063.729348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076063.729350:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076063.729352:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1769 00000100:00080000:1.0:1769076063.729354:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076063.729356:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076063.729359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076063.729368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913226368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076063.729377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076063.729380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076063.729381:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076063.729385:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1769 00000100:00080000:1.0:1769076063.729388:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076063.729391:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076063.729394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076063.729402:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913226496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076063.729408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076063.729411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076063.729413:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076063.729416:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1769 00000100:00080000:1.0:1769076063.729418:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076063.729420:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076063.729423:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076063.729430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913226624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076063.729435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076063.729437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076063.729438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076063.729444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1769 00000100:00080000:1.0:1769076063.729448:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076063.729452:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076063.729455:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076063.729464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913226752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076063.921180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076064.433223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076064.433223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076064.433339:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076064.753196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076064.945166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076065.457236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076065.457312:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076065.457341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076065.777285:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076065.969166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076066.481190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076066.481191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076066.481294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076066.801184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076066.993168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076067.505247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076067.505255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076067.825288:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076068.017249:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076068.529216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076068.529264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076068.849200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076069.041280:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076069.553214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076069.553245:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076069.873228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076070.065158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076070.577218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076070.577259:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076070.897283:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076071.089177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076071.601178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076071.601217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076071.921190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076072.113194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076072.625183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076072.625236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076072.945191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076073.137196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076073.649185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076073.649207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076073.969176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076073.969223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.969239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076073.969248:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.969265:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1779 00000100:00080000:3.0:1769076073.969284:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.969297:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076073.969376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913227776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.969424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.969431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076073.969435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.969443:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1779 00000100:00080000:3.0:1769076073.969451:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.969459:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076073.969487:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.969518:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913227904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.969547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.969555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076073.969559:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.969568:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1779 00000100:00080000:3.0:1769076073.969578:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.969586:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076073.969596:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076073.969628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913228032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.969703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.969712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076073.969717:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.969732:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1779 00000100:00080000:3.0:1769076073.969749:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.969762:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076073.969776:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076073.969819:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913228160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.969866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.969876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076073.969882:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.969894:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1779 00000100:00080000:3.0:1769076073.969910:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.969923:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076073.969939:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.969994:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913228288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.970035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.970046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076073.970095:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.970105:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1779 00000100:00080000:3.0:1769076073.970115:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.970125:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076073.970140:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.970177:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913228416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.970232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.970243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076073.970252:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.970272:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1779 00000100:00080000:3.0:1769076073.970290:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.970309:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076073.970329:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.970363:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913228544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076074.161156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076074.673182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076074.673198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076074.993193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076075.185150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076075.697222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076075.697258:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076076.017217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076076.209182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076076.721249:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076076.721272:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076077.041268:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076077.233154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076077.745210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076077.745267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076077.745334:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076078.065203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076078.257228:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076078.769203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076078.769205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076078.769313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076079.089198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076079.281139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076079.793179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076079.793274:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076080.113188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076080.305158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076080.817156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076080.817201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076081.137138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076081.329213:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076081.841225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076081.841241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076082.161198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076082.353180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076082.865253:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076082.865254:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076083.185208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076083.377139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076083.889229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076083.889230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076083.889364:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076083.889385:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076084.209184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076084.401197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076084.913190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076084.913218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076084.913250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076085.233178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076085.425182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076085.937161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076085.937288:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076086.257162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076086.449174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076086.961209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076086.961209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076086.961396:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076087.281165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076087.473206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076087.985181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076087.985226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076087.985277:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076088.305198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076088.497227:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076089.009185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076089.009205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076089.009267:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076089.329188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076089.521189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076090.033200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076090.033215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076090.033370:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076090.353163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076090.545191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076091.057191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076091.057306:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076091.057440:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076091.377169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076091.569159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076092.081170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076092.081226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076092.081421:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076092.401210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076092.593152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076093.105129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076093.105129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076093.425167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076093.617193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076094.129179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076094.129179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076094.449247:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076094.641157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076095.153212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076095.153224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076095.473212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076096.177152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076096.177173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076096.497210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076097.201179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076097.201219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076097.201323:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076097.521182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076098.225210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076098.225291:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076098.545200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076099.249247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076099.249289:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076099.569208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076100.273254:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076100.273352:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076100.593265:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076101.297177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076101.297218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076101.617312:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076102.321157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076102.321246:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076102.641136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076103.345224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076103.345318:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076103.665295:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076104.369235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076104.369294:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076104.689237:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076104.689321:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076104.689335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076104.689341:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076104.689356:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1810 00000100:00080000:3.0:1769076104.689374:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076104.689383:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076104.689442:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913233152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076104.689499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076104.689505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076104.689507:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076104.689513:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1810 00000100:00080000:3.0:1769076104.689523:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076104.689528:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076104.689543:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076104.689558:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913233280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076104.689573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076104.689579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076104.689581:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076104.689587:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1810 00000100:00080000:3.0:1769076104.689591:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076104.689596:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076104.689605:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076104.689626:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913233408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076104.689643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076104.689647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076104.689650:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076104.689654:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1810 00000100:00080000:3.0:1769076104.689658:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076104.689662:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076104.689669:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076104.689687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913233536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076104.689698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076104.689704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076104.689707:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076104.689711:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1810 00000100:00080000:3.0:1769076104.689716:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076104.689726:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076104.689736:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076104.689753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913233664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076104.689765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076104.689769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076104.689773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076104.689778:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1810 00000100:00080000:3.0:1769076104.689785:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076104.689795:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076104.689809:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076104.689837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913233792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076104.689880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076104.689888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076104.689897:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076104.689912:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1810 00000100:00080000:3.0:1769076104.689917:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076104.689924:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076104.689928:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076104.689941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913233920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076105.393145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076105.393167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076105.393296:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076105.393373:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076105.713212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076106.417199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076106.417232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076106.417267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076106.737212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076107.441182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076107.441234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076107.441325:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076107.761198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076108.465217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076108.465217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076108.785180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076109.489287:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076109.489298:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076109.809251:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076110.513161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076110.513202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076110.833152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076111.537169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076111.537240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076111.857169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076112.561140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076112.561152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076112.881170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076113.585310:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076113.585310:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076113.905204:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076114.609196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076114.609198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076114.929243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076114.929262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076114.929274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076114.929281:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076114.929296:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1820 00000100:00080000:3.0:1769076114.929313:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076114.929325:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076114.929381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913234944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076114.929422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076114.929429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076114.929433:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076114.929441:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1820 00000100:00080000:3.0:1769076114.929448:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076114.929456:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076114.929473:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076114.929497:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913235072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076114.929517:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076114.929524:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076114.929528:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076114.929535:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1820 00000100:00080000:3.0:1769076114.929541:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076114.929547:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076114.929557:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076114.929575:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913235200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076114.929591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076114.929596:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076114.929599:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076114.929605:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1820 00000100:00080000:3.0:1769076114.929612:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076114.929617:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076114.929626:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076114.929642:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913235328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076114.929669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076114.929675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076114.929678:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076114.929684:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1820 00000100:00080000:3.0:1769076114.929690:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076114.929697:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076114.929706:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076114.929723:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913235456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076114.929740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076114.929746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076114.929750:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076114.929757:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1820 00000100:00080000:3.0:1769076114.929762:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076114.929769:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076114.929778:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076114.929794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913235584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076114.929809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076114.929815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076114.929818:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076114.929825:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1820 00000100:00080000:3.0:1769076114.929831:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076114.929838:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076114.929846:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076114.929864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913235712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076115.633256:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076115.633257:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076115.953200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076116.657165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076116.657173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076116.977175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076117.681123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076117.681171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076118.001161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076118.705239:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076118.705293:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076119.025173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076119.729237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076119.729293:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076120.049212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076120.753223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076120.753233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076121.073232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076121.777222:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076121.777228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076121.777381:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076122.097160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076122.801219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076122.801283:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076122.801369:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076123.121226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769076123.121493:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769076123.121563:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076123.121590:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076123.121661:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076123.121673:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076123.121685:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076123.121694:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076123.121706:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076123.121725:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:3.0:1769076123.825241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076123.825254:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076123.825389:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076124.145190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076124.849196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076124.849205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076124.849302:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076125.169163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076125.873176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076125.873230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076125.873331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076126.193177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076126.897139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076126.897166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076126.897226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076126.897273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076127.217170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076127.921184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076127.921243:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076127.921375:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076128.241171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076128.945188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076128.945249:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076128.945288:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076128.945322:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076129.265189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00010000:00010000:1.0:1769076129.969228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076129.969232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076129.969271:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076130.289231:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076130.993204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076130.993214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076130.993258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076131.313152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076132.017146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076132.017192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076132.017430:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076132.337183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076133.041186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076133.041259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076133.361246:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076134.065210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076134.065281:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076134.385173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076135.089145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076135.089192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076135.089201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076135.409181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076136.113197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076136.113219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076136.433327:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076137.137197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076137.137226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076137.457218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076138.161191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076138.161201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076138.481154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076139.185217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076139.185220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076139.505167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076140.209201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076140.209217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076140.529191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076141.233131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076141.233141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076141.553149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076142.257191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076142.257191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076142.577169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076143.281198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076143.281228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076143.601116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076144.305164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076144.305188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076144.625198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076145.329198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076145.649221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076145.650508:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913240960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076145.650519:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076145.650523:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076145.650527:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913240960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076145.650565:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076145.650591:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076145.650603:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076145/real 1769076145] req@ffff9350103f8380 x1855008913241088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076145.650618:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913241088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076145.650628:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076145.650633:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076145.650636:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076145.841222:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076146.353246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076146.353299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076146.353314:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076146.673201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076146.865162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076147.377128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076147.377153:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076147.697172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076147.889168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076148.401256:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076148.401268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076148.401301:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076148.721145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076148.913187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076149.425181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076149.425215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076149.425348:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076149.745270:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076149.937178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076150.449159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076150.449242:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076150.449245:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076150.769184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076150.769479:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913241216:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769076150.769591:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076150.769679:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076150/real 1769076150] req@ffff9350103fb100 x1855008913241216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076150.769740:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913241216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076150.769761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076150.769777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076150.769788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913241216:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076150.769829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913241344:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076150.769870:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076150.769896:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076150/real 1769076150] req@ffff9350103fbb80 x1855008913241344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076150.769961:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913241344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076150.770001:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076150.770008:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076150.770015:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913241344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076150.770026:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913241472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076150.770098:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076150.770114:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076150/real 1769076150] req@ffff9350103f9f80 x1855008913241472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076150.770130:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913241472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076150.770149:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076150.770153:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076150.770157:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913241472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076150.770164:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913241600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076150.770187:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076150.770198:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076150/real 1769076150] req@ffff9350103faa00 x1855008913241600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076150.770212:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913241600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076150.770222:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076150.770227:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076150.770230:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913241600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076150.770237:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076150.770254:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076150.770264:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076150/real 1769076150] req@ffff9350103fb480 x1855008913241728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076150.770277:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913241728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076150.770287:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076150.770291:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076150.770296:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076150.770303:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076150.770321:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076150.770331:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076150/real 1769076150] req@ffff9350103f8380 x1855008913241856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076150.770344:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913241856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076150.770354:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076150.770358:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076150.770391:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076150.770417:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076150.770440:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076150.770452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076150/real 1769076150] req@ffff9350103f8700 x1855008913241984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076150.770467:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913241984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076150.770477:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076150.770481:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076150.770486:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913241984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076150.961240:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076151.473195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076151.473195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076151.473277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076151.793191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076151.985169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076152.497210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076152.497210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076152.497263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076152.817252:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076153.009171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076153.521207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076153.521207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076153.521247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076153.841162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076154.033179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076154.545156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076154.545188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076154.545195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076154.865149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076155.057155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076155.569216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076155.569219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076155.569372:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076155.889123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076155.889499:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913242112:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769076155.889619:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076155.889691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff9350103f9c00 x1855008913242112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076155.889780:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913242112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076155.889809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076155.889820:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076155.889829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913242112:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076155.889845:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913242240:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076155.889889:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076155.889908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff9350103f9500 x1855008913242240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076155.889926:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913242240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076155.889938:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076155.889942:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076155.889947:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913242240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076155.889983:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913242368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076155.890005:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076155.890017:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff9350103fad80 x1855008913242368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076155.890032:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913242368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076155.890042:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076155.890045:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076155.890101:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913242368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076155.890109:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913242496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076155.890134:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076155.890148:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff9350103fb800 x1855008913242496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076155.890163:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913242496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076155.890173:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076155.890177:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076155.890181:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913242496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076155.890188:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913242624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076155.890207:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076155.890219:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff9350103f9180 x1855008913242624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076155.890233:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913242624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076155.890243:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076155.890247:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076155.890250:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913242624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076155.890257:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913242752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076155.890303:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076155.890321:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff9350103f8700 x1855008913242752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076155.890337:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913242752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076155.890347:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076155.890352:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076155.890356:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913242752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076155.890363:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913242880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076155.890386:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076155.890397:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff9350103f8380 x1855008913242880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076155.890411:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913242880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076155.890422:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076155.890426:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076155.890430:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913242880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076156.081266:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076156.593175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076156.593189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076156.593327:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076156.913188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076157.105149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076157.617208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076157.617243:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076157.617281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076157.937209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076158.129154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076158.641205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076158.641210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076158.641331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076158.961145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076159.153197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076159.665134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076159.665138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076159.665195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076159.985140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076160.177168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076160.689242:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076160.689304:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076160.689321:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076161.009166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076161.009563:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913243008:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769076161.009721:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076161.009802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076161/real 1769076161] req@ffff9350103fb480 x1855008913243008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076161.009842:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913243008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076161.009881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076161.009899:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076161.009930:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913243008:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076161.009991:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913243136:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076161.010107:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076161.010135:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076161/real 1769076161] req@ffff9350103faa00 x1855008913243136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076161.010154:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913243136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076161.010168:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076161.010172:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076161.010177:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913243136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076161.010187:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913243264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076161.010250:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076161.010266:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076161/real 1769076161] req@ffff9350103f9f80 x1855008913243264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076161.010282:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913243264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076161.010293:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076161.010298:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076161.010302:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913243264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076161.010310:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913243392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076161.010330:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076161.010341:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076161/real 1769076161] req@ffff9350103fbb80 x1855008913243392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076161.010355:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913243392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076161.010367:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076161.010371:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076161.010375:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913243392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076161.010382:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913243520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076161.010401:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076161.010412:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076161/real 1769076161] req@ffff9350103fb100 x1855008913243520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076161.010425:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913243520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076161.010437:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076161.010443:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076161.010451:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913243520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076161.010464:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913243648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076161.010502:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076161.010524:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076161/real 1769076161] req@ffff9350103f8380 x1855008913243648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076161.010551:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913243648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076161.010569:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076161.010576:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076161.010583:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913243648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076161.010616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913243776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076161.010646:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076161.010661:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076161/real 1769076161] req@ffff9350103f8700 x1855008913243776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076161.010676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913243776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076161.010687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076161.010691:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076161.010695:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913243776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076161.201172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076161.713275:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076161.713275:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076161.713349:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076162.033206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076162.225147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076162.737232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076162.737283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076162.737343:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076163.057299:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076163.249222:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076163.761251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076163.761252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076163.761331:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076164.081213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076164.273336:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076164.785232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076164.785280:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076164.785397:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076165.105202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076165.297172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076165.809238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076165.809243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076165.809325:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076166.129228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076166.129297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076166.129307:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076166.129313:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076166.129324:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1871 00000100:00080000:3.0:1769076166.129340:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076166.129351:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076166.129411:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913243904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076166.129457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076166.129464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076166.129468:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076166.129474:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1871 00000100:00080000:3.0:1769076166.129479:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076166.129496:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076166.129509:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076166.129527:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913244032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076166.129540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076166.129545:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076166.129547:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076166.129552:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1871 00000100:00080000:3.0:1769076166.129556:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076166.129560:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076166.129570:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076166.129583:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913244160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076166.129598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076166.129602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076166.129604:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076166.129608:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1871 00000100:00080000:3.0:1769076166.129612:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076166.129620:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076166.129625:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913243904:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076166.129627:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076166.129652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913244288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076166.129664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076166.129691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076166.129695:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076166.129701:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1871 00000100:00080000:3.0:1769076166.129710:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076166.129715:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076166.129723:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076166.129739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913244416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076166.129752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076166.129756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076166.129758:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076166.129771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076166.129782:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1871 00000100:00080000:3.0:1769076166.129790:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076166.129796:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076166.129821:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076166.129837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913244544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076166.129849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076166.129854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076166.129859:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076166.129867:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076166/real 1769076166] req@ffff935002a7c380 x1855008913243904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076166.129876:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1871 00000100:00080000:3.0:1769076166.129894:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076166.129931:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076166.129954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076166.129978:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913243904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076166.129994:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913244672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076166.129997:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076166.130017:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076166.130035:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913243904:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076166.130048:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076166.130178:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076166.130201:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076166/real 1769076166] req@ffff935002a7ca80 x1855008913244032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076166.130218:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913244032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076166.130230:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076166.130235:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076166.130244:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076166.130282:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076166.130305:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076166.130318:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076166/real 1769076166] req@ffff935002a7c700 x1855008913244160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076166.130333:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913244160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076166.130350:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076166.130355:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076166.130359:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076166.130367:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076166.130385:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076166.130396:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076166/real 1769076166] req@ffff935002a7fb80 x1855008913244288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076166.130410:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913244288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076166.130420:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076166.130423:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076166.130427:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076166.130434:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913244416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076166.130452:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076166.130463:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076166/real 1769076166] req@ffff935002a7f100 x1855008913244416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076166.130478:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913244416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076166.130487:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076166.130491:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076166.130496:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913244416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076166.130503:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913244544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076166.130521:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076166.130531:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076166/real 1769076166] req@ffff935002a7e300 x1855008913244544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076166.130545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913244544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076166.130555:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076166.130559:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076166.130562:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913244544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076166.130569:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913244672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076166.130587:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076166.130597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076166/real 1769076166] req@ffff935002a7d180 x1855008913244672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076166.130612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913244672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076166.130621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076166.130624:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076166.130628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913244672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076166.321267:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076166.833167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076166.833178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076166.833221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076167.153188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076167.345185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076167.857259:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076167.857331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076167.857334:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076168.177264:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076168.369246:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076168.881281:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076168.881301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076168.881359:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076169.201215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076169.393262:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076169.905233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076169.905233:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076169.905348:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076170.225284:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076170.417201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076170.929147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076170.929169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076170.929213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076170.929232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076171.249151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076171.249159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076171.249166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076171.249171:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076171.249181:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1876 00000100:00080000:1.0:1769076171.249192:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076171.249198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076171.249239:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913244800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076171.249266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076171.249271:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076171.249273:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076171.249276:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1876 00000100:00080000:1.0:1769076171.249278:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076171.249280:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076171.249290:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076171.249301:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913244928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076171.249310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076171.249313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076171.249314:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076171.249319:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1876 00000100:00100000:2.0:1769076171.249321:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913244800:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076171.249322:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076171.249328:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076171.249333:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076171.249341:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913245056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076171.249347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076171.249350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076171.249351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076171.249354:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1876 00000100:00080000:1.0:1769076171.249357:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076171.249359:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076171.249363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076171.249370:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913245184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076171.249376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076171.249379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076171.249380:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076171.249382:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1876 00000100:00080000:1.0:1769076171.249385:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076171.249388:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076171.249392:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076171.249397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076171.249406:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913245312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076171.249412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076171.249413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076171.249415:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076171.249417:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1876 00000100:00080000:1.0:1769076171.249420:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076171.249423:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076171.249427:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076171.249431:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076171/real 1769076171] req@ffff9350100cb800 x1855008913244800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076171.249436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913245440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076171.249455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076171.249457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076171.249459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076171.249464:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1876 00000100:00100000:2.0:1769076171.249466:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913244800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076171.249467:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076171.249473:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076171.249479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076171.249491:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913245568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076171.249491:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076171.249502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076171.249510:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913244800:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076171.249533:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244928:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076171.249555:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076171.249565:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076171/real 1769076171] req@ffff9350100caa00 x1855008913244928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076171.249580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913244928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076171.249591:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076171.249593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076171.249596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913244928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076171.249600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076171.249613:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076171.249620:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076171/real 1769076171] req@ffff9350100c9500 x1855008913245056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076171.249632:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913245056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076171.249638:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076171.249640:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076171.249642:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076171.249646:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076171.249656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076171.249662:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076171/real 1769076171] req@ffff9350100c9880 x1855008913245184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076171.249669:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913245184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076171.249674:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076171.249676:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076171.249679:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076171.249683:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913245312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076171.249693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076171.249698:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076171/real 1769076171] req@ffff9350100c8e00 x1855008913245312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076171.249705:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913245312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076171.249711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076171.249712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076171.249714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913245312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076171.249718:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913245440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076171.249727:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076171.249732:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076171/real 1769076171] req@ffff9350100ca680 x1855008913245440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076171.249740:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913245440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076171.249745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076171.249747:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076171.249748:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913245440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076171.249752:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913245568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076171.249760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076171.249766:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076171/real 1769076171] req@ffff9350100ca300 x1855008913245568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076171.249773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913245568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076171.249778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076171.249780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076171.249782:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913245568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076171.441145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076171.953183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076171.953269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076171.953277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076172.273292:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076172.465200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076172.977178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076172.977186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076172.977221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076173.297181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076173.489183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076174.001120:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076174.001122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076174.001143:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076174.321106:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076174.513189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076175.025143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076175.025168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076175.025250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076175.345183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076175.537186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076176.049182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076176.049190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076176.049281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076176.369285:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076176.369656:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913245696:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769076176.369814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076176.369923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076176/real 1769076176] req@ffff9350103f9180 x1855008913245696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076176.370020:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913245696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076176.370037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076176.370044:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076176.370095:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913245696:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076176.370108:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076176.370147:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076176.370165:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076176/real 1769076176] req@ffff9350103fb800 x1855008913245824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076176.370182:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913245824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076176.370194:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076176.370198:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076176.370203:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076176.370239:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076176.370261:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076176.370273:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076176/real 1769076176] req@ffff9350103fad80 x1855008913245952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076176.370288:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913245952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076176.370298:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076176.370302:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076176.370306:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913245952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076176.370313:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913246080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076176.370330:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076176.370342:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076176/real 1769076176] req@ffff9350103f9500 x1855008913246080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076176.370357:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913246080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076176.370366:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076176.370370:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076176.370373:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913246080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076176.370380:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913246208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076176.370398:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076176.370409:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076176/real 1769076176] req@ffff9350103f9c00 x1855008913246208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076176.370423:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913246208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076176.370433:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076176.370437:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076176.370440:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913246208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076176.370447:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913246336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076176.370464:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076176.370474:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076176/real 1769076176] req@ffff9350103f8700 x1855008913246336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076176.370487:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913246336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076176.370498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076176.370502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076176.370506:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913246336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076176.370513:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913246464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076176.370529:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076176.370540:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076176/real 1769076176] req@ffff9350103f8380 x1855008913246464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076176.370552:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913246464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076176.370562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076176.370566:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076176.370570:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913246464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076176.561190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076177.073184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076177.073221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076177.073231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076177.393201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076177.585161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076178.097196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076178.097207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076178.097282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076178.417182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076178.609172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076179.121190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076179.121231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076179.121262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076179.441167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076179.633190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076180.145243:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076180.145243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076180.145330:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076180.465128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076180.657237:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076181.169155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076181.169166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076181.169346:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076181.489177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076181.489221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076181.489229:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076181.489236:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076181.489245:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1887 00000100:00080000:2.0:1769076181.489258:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076181.489266:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076181.489322:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913246592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076181.489357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076181.489360:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076181.489363:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076181.489367:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1887 00000100:00080000:2.0:1769076181.489370:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076181.489374:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076181.489385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076181.489399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913246720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076181.489414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076181.489423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076181.489425:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076181.489429:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1887 00000100:00080000:2.0:1769076181.489432:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076181.489435:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076181.489441:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076181.489454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913246848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076181.489499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076181.489504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076181.489507:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076181.489513:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1887 00000100:00080000:2.0:1769076181.489519:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076181.489524:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076181.489530:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076181.489552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913246976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076181.489578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076181.489583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076181.489585:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076181.489591:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1887 00000100:00080000:2.0:1769076181.489596:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076181.489600:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076181.489606:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076181.489625:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913247104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076181.489642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076181.489656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076181.489661:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076181.489674:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1887 00000100:00080000:2.0:1769076181.489693:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076181.489698:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076181.489704:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076181.489737:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913247232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076181.489754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076181.489761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076181.489765:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076181.489772:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1887 00000100:00080000:2.0:1769076181.489781:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076181.489790:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076181.489796:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076181.489806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913247360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076181.681204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076182.193119:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076182.193139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076182.193209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076182.513150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076182.705149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076183.217204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076183.217253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076183.217308:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076183.537125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076183.729206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076184.241197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076184.241232:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076184.241449:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076184.561131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076184.753175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076185.265150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076185.265156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076185.265194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076185.585124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076185.777222:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076186.289221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076186.289234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076186.289341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076186.609228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076186.609229:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076186.609247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076186.609256:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076186.609271:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1892 00000100:00080000:2.0:1769076186.609286:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076186.609296:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076186.609353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913247488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076186.609395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076186.609401:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076186.609403:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076186.609409:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1892 00000100:00080000:2.0:1769076186.609413:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076186.609417:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076186.609431:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076186.609445:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913247616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076186.609459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076186.609464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076186.609466:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076186.609472:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1892 00000100:00080000:2.0:1769076186.609476:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076186.609480:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076186.609487:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076186.609506:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913247744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076186.609540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076186.609545:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076186.609547:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076186.609560:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1892 00000100:00080000:2.0:1769076186.609565:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076186.609570:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076186.609576:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076186.609592:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913247872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076186.609602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076186.609608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076186.609612:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076186.609617:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1892 00000100:00080000:2.0:1769076186.609622:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076186.609627:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076186.609634:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076186.609647:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913248000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076186.609659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076186.609663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076186.609666:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076186.609671:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1892 00000100:00080000:2.0:1769076186.609678:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076186.609684:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076186.609693:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076186.609727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913248128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076186.609741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076186.609746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076186.609749:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076186.609755:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1892 00000100:00080000:2.0:1769076186.609789:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076186.609806:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076186.609816:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076186.609839:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913248256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076186.801206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076187.313185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076187.313188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076187.313280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076187.633315:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076187.825236:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076188.337135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076188.337148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076188.337335:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076188.657197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076188.849301:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076189.361172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076189.361202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076189.361291:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076189.681381:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076189.681582:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076189.681676:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076189.681706:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076189.681801:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076189.681818:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076189.681834:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076189.681852:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076189.681871:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076189.681891:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769076189.873175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076190.385186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076190.385218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076190.385265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076190.705244:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076190.897177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076191.409206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076191.409251:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076191.409422:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076191.729250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076191.729265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076191.729272:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076191.729290:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1897 00000100:00080000:3.0:1769076191.729309:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076191.729319:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076191.729336:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076191.729380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913248384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076191.729424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076191.729430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076191.729433:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076191.729439:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1897 00000100:00080000:3.0:1769076191.729443:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076191.729449:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076191.729464:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076191.729479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913248512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076191.729492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076191.729497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076191.729500:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076191.729505:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1897 00000100:00080000:3.0:1769076191.729509:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076191.729513:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076191.729520:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076191.729533:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913248640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076191.729547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076191.729551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076191.729553:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076191.729561:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1897 00000100:00080000:3.0:1769076191.729566:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076191.729577:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076191.729587:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076191.729601:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913248768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076191.729614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076191.729620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076191.729623:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076191.729628:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1897 00000100:00080000:3.0:1769076191.729633:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076191.729639:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076191.729644:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076191.729658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913248896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076191.729669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076191.729699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076191.729703:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076191.729716:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1897 00000100:00080000:3.0:1769076191.729725:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076191.729730:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076191.729738:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076191.729755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913249024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076191.729768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076191.729772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076191.729775:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076191.729780:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1897 00000100:00080000:3.0:1769076191.729786:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076191.729790:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076191.729798:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076191.729816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913249152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076191.921212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076192.433229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076192.433246:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076192.433305:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076192.753383:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076192.945215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076193.457191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076193.457219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076193.457332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076193.777202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076193.969203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076194.481224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076194.481229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076194.481298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076194.801180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076194.993145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076195.505181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076195.505235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076195.505329:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076195.825180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076196.017187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076196.529245:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076196.529249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076196.529466:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076196.849205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076196.849216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.849235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076196.849243:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.849263:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1902 00000100:00080000:3.0:1769076196.849284:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.849297:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076196.849363:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913249280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076196.849421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.849430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076196.849434:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.849443:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1902 00000100:00080000:3.0:1769076196.849452:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.849463:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076196.849483:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076196.849509:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913249408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076196.849542:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.849549:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076196.849553:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.849563:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1902 00000100:00080000:3.0:1769076196.849569:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.849574:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076196.849584:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076196.849634:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913249536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076196.849648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.849654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076196.849658:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.849674:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1902 00000100:00080000:3.0:1769076196.849686:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.849695:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076196.849706:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076196.849726:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913249664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076196.849741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.849751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076196.849758:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.849784:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1902 00000100:00080000:3.0:1769076196.849800:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.849810:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076196.849818:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076196.849845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913249792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076196.849861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.849869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076196.849872:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.849887:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1902 00000100:00080000:3.0:1769076196.849896:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.849908:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076196.849920:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076196.849943:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913249920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076196.849958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.849962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076196.849964:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.849969:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1902 00000100:00080000:3.0:1769076196.849974:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.849978:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076196.849983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076196.849995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913250048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076197.041213:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076197.553201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076197.553255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076197.553260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076197.873217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076198.065148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076198.577188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076198.577227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076198.577266:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076198.577293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076198.897166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076199.089178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076199.601228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076199.601228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076199.601282:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076199.601283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076199.921190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076200.113240:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076200.625238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076200.625260:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076200.625303:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076200.625326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076200.945246:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076201.137133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076201.649143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076201.649201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076201.649259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076201.969104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.969117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076201.969121:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.969134:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1907 00000100:00080000:3.0:1769076201.969148:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.969156:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076201.969205:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913250176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.969239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.969243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.969246:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.969252:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1907 00000100:00080000:3.0:1769076201.969255:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.969261:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076201.969262:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000008:00100000:3.0:1769076201.969275:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.969298:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913250304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.969315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.969320:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.969321:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.969329:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1907 00000100:00080000:3.0:1769076201.969332:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.969336:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076201.969342:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076201.969355:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913250432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.969382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.969385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076201.969386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.969390:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1907 00000100:00080000:3.0:1769076201.969393:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.969398:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076201.969405:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076201.969416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913250560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.969423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.969426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.969428:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.969432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1907 00000100:00080000:3.0:1769076201.969435:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.969439:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076201.969444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.969453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913250688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.969463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.969466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.969467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.969470:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1907 00000100:00080000:3.0:1769076201.969473:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.969481:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076201.969494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.969532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913250816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.969559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.969564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076201.969566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.969571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1907 00000100:00080000:3.0:1769076201.969578:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.969583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076201.969594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.969616:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913250944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076202.161179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076202.673193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076202.673204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076202.673264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076202.993214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076203.185221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076203.697147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076203.697215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076203.697240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076204.017181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076204.209144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076204.721160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076204.721168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076204.721242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076205.041179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076205.233155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076205.745187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076205.745191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076205.745267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076206.065176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076206.257195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076206.769166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076206.769222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076206.769270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076207.089170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076207.089172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076207.089187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076207.089194:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076207.089206:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1912 00000100:00080000:3.0:1769076207.089221:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076207.089232:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076207.089281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913251072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076207.089340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076207.089345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076207.089348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076207.089353:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1912 00000100:00080000:3.0:1769076207.089357:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076207.089361:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076207.089374:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076207.089391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913251200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076207.089405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076207.089409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076207.089412:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076207.089445:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1912 00000100:00080000:3.0:1769076207.089459:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076207.089471:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076207.089479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076207.089498:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913251328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076207.089512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076207.089517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076207.089519:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076207.089524:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1912 00000100:00080000:3.0:1769076207.089529:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076207.089532:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076207.089538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076207.089553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913251456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076207.089576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076207.089581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076207.089584:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076207.089590:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1912 00000100:00080000:3.0:1769076207.089596:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076207.089601:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076207.089607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076207.089624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913251584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076207.089641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076207.089656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076207.089663:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076207.089676:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1912 00000100:00080000:3.0:1769076207.089690:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076207.089695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076207.089704:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076207.089728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913251712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076207.089755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076207.089762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076207.089769:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076207.089779:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1912 00000100:00080000:3.0:1769076207.089784:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076207.089796:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076207.089804:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076207.089821:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913251840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076207.281151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076207.793185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076207.793230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076207.793258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076208.113231:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076208.305185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076208.817170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076208.817192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076208.817257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076209.137225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076209.329179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076209.841272:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076209.841309:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076209.841349:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076210.161254:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076210.353254:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076210.865194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076210.865248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076210.865251:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076211.185215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076211.377196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076211.889191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076211.889227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076211.889239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076211.889291:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076212.209102:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076212.209168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076212.209179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076212.209185:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076212.209220:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1917 00000100:00080000:2.0:1769076212.209233:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076212.209240:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076212.209279:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913251968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076212.209306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076212.209309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076212.209310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076212.209313:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1917 00000100:00080000:2.0:1769076212.209316:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076212.209318:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076212.209326:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076212.209351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913252096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076212.209364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076212.209367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076212.209371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076212.209375:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1917 00000100:00080000:2.0:1769076212.209381:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076212.209385:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076212.209389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076212.209397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913252224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076212.209403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076212.209406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076212.209407:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076212.209409:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1917 00000100:00080000:2.0:1769076212.209412:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076212.209414:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076212.209417:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076212.209425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913252352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076212.209430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076212.209432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076212.209434:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076212.209437:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1917 00000100:00080000:2.0:1769076212.209441:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076212.209444:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076212.209447:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076212.209460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913252480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076212.209468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076212.209471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076212.209473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076212.209475:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1917 00000100:00080000:2.0:1769076212.209478:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076212.209480:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076212.209483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076212.209492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913252608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076212.209502:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076212.209509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076212.209513:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076212.209517:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1917 00000100:00080000:2.0:1769076212.209520:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076212.209525:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076212.209532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076212.209543:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913252736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076212.401189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076212.913180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076212.913241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076212.913262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076213.233211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076213.425163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076213.937170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076213.937232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076213.937262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076213.937338:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076214.257148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076214.449195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076214.961213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076214.961252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076214.961262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076215.281163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076215.473191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076215.985144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076215.985221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076215.985247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076216.305228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076216.497159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076217.009169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076217.009198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076217.009306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076217.329204:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076217.329523:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913252864:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769076217.329672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076217.329769:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076217/real 1769076217] req@ffff9350103fb100 x1855008913252864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076217.329810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913252864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076217.329840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076217.329855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076217.329866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913252864:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076217.329909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913252992:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076217.329952:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076217.329976:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076217/real 1769076217] req@ffff9350103fbb80 x1855008913252992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076217.329994:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913252992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076217.330007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076217.330011:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076217.330016:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913252992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076217.330024:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913253120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076217.330045:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076217.330093:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076217/real 1769076217] req@ffff9350103f9f80 x1855008913253120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076217.330108:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913253120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076217.330119:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076217.330123:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076217.330127:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913253120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076217.330134:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913253248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076217.330158:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076217.330170:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076217/real 1769076217] req@ffff9350103faa00 x1855008913253248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076217.330185:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913253248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076217.330205:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076217.330209:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076217.330214:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913253248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076217.330222:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913253376:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076217.330241:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076217.330284:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076217/real 1769076217] req@ffff9350103fb480 x1855008913253376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076217.330307:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913253376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076217.330318:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076217.330322:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076217.330327:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913253376:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076217.330334:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913253504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076217.330362:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076217.330384:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076217/real 1769076217] req@ffff9350103f8380 x1855008913253504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076217.330399:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913253504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076217.330409:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076217.330413:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076217.330417:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913253504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076217.330443:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913253632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076217.330462:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076217.330474:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076217/real 1769076217] req@ffff9350103f8700 x1855008913253632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076217.330489:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913253632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076217.330499:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076217.330502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076217.330506:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913253632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076217.521222:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076218.033197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076218.033322:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076218.033356:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076218.353257:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076218.545185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076219.057176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076219.057183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076219.057211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076219.377131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076219.569169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076220.081120:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076220.081162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076220.081175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076220.401127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076220.593137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076221.105109:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076221.105110:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076221.105164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076221.425229:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076221.617168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076222.129226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076222.129257:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076222.129337:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076222.449195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076222.449233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076222.449273:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076222.449280:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076222.449293:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1928 00000100:00080000:2.0:1769076222.449310:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076222.449321:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076222.449375:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913253760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076222.449415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076222.449421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076222.449423:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076222.449429:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1928 00000100:00080000:2.0:1769076222.449434:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076222.449439:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076222.449454:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076222.449471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913253888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076222.449506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076222.449519:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076222.449522:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076222.449530:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1928 00000100:00080000:2.0:1769076222.449537:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076222.449544:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076222.449555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076222.449579:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913254016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076222.449598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076222.449604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076222.449625:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076222.449641:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1928 00000100:00080000:2.0:1769076222.449650:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076222.449657:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076222.449666:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076222.449687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913254144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076222.449702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076222.449712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076222.449719:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076222.449735:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1928 00000100:00080000:2.0:1769076222.449759:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076222.449774:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076222.449787:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076222.449833:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913254272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076222.449847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076222.449850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076222.449852:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076222.449856:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1928 00000100:00080000:2.0:1769076222.449861:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076222.449869:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076222.449901:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076222.449923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913254400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076222.449960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076222.449969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076222.449974:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076222.449988:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1928 00000100:00080000:2.0:1769076222.450003:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076222.450017:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076222.450045:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076222.450109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913254528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076222.641201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076223.153190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076223.153236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076223.153374:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076223.473206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076223.665198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076224.177179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076224.177301:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076224.177374:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076224.497284:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076224.689196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076225.201238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076225.201278:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076225.201347:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076225.521265:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076225.713191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076226.225220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076226.225256:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076226.225283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076226.545232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076226.737228:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076227.249193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076227.249193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076227.569127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076227.569147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076227.569154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076227.569158:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076227.569167:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1933 00000100:00080000:3.0:1769076227.569185:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076227.569192:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076227.569228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913254656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076227.569251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076227.569255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076227.569256:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076227.569260:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1933 00000100:00080000:3.0:1769076227.569262:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076227.569265:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076227.569275:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076227.569288:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913254784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076227.569304:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076227.569306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076227.569308:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076227.569311:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1933 00000100:00080000:3.0:1769076227.569314:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076227.569317:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076227.569321:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076227.569329:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913254912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076227.569357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076227.569362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076227.569365:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076227.569369:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1933 00000100:00080000:3.0:1769076227.569373:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076227.569378:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076227.569385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076227.569399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913255040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076227.569421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076227.569427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076227.569430:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076227.569438:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1933 00000100:00080000:3.0:1769076227.569454:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076227.569460:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076227.569468:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076227.569489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913255168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076227.569526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076227.569531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076227.569538:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076227.569546:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1933 00000100:00080000:3.0:1769076227.569556:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076227.569567:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076227.569580:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076227.569604:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913255296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076227.569632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076227.569638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076227.569646:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076227.569659:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1933 00000100:00080000:3.0:1769076227.569667:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076227.569672:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076227.569680:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076227.569715:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913255424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076227.761171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076228.273214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076228.273268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076228.273293:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076228.273298:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076228.593167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076228.785230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076229.297230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076229.297252:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076229.297298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076229.617170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076229.809178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076230.321228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076230.321291:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076230.321402:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076230.641202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076230.833160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076231.345164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076231.345175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076231.345181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076231.665134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076231.857135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076232.369210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076232.369218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076232.369307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076232.689216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076232.689221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076232.689244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076232.689253:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076232.689274:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1938 00000100:00080000:1.0:1769076232.689294:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076232.689328:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076232.689397:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913255552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076232.689441:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076232.689446:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076232.689449:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076232.689455:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1938 00000100:00080000:1.0:1769076232.689461:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076232.689466:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076232.689482:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076232.689501:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913255680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076232.689517:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076232.689522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076232.689525:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076232.689546:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1938 00000100:00080000:1.0:1769076232.689556:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076232.689573:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076232.689580:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076232.689592:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913255808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076232.689607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076232.689613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076232.689617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076232.689623:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1938 00000100:00080000:1.0:1769076232.689628:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076232.689635:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076232.689654:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076232.689671:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913255936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076232.689685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076232.689693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076232.689701:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076232.689711:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1938 00000100:00080000:1.0:1769076232.689720:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076232.689728:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076232.689737:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076232.689753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913256064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076232.689781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076232.689809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076232.689812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076232.689821:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1938 00000100:00080000:1.0:1769076232.689829:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076232.689845:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076232.689869:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076232.689905:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913256192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076232.689930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076232.689940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076232.689944:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076232.689951:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1938 00000100:00080000:1.0:1769076232.689955:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076232.689959:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076232.689965:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076232.690008:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913256320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076232.881153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076233.393207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076233.393216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076233.393263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076233.713291:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076233.905183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076234.417167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076234.417242:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076234.417337:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076234.737159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076234.929182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076235.441191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076235.441219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076235.441266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076235.761203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076235.953194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076236.465156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076236.465267:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076236.785165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076236.977150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076237.489226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076237.489277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076237.489279:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769076237.809317:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076237.809337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076237.809345:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076237.809363:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1943 00000100:00100000:3.0:1769076237.809364:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076237.809385:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076237.809402:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076237.809464:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913256448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076237.809513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076237.809521:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076237.809525:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076237.809533:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1943 00000100:00080000:1.0:1769076237.809540:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076237.809548:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076237.809565:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076237.809608:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913256576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076237.809645:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076237.809654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076237.809658:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076237.809671:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1943 00000100:00080000:1.0:1769076237.809679:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076237.809689:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076237.809700:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076237.809749:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913256704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076237.809776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076237.809789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076237.809794:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076237.809806:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1943 00000100:00080000:1.0:1769076237.809818:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076237.809829:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076237.809843:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076237.809950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913256832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076237.810003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076237.810011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076237.810016:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076237.810034:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1943 00000100:00080000:1.0:1769076237.810137:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076237.810179:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076237.810199:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076237.810242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913256960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076237.810356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076237.810376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076237.810382:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076237.810411:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1943 00000100:00080000:1.0:1769076237.810426:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076237.810453:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076237.810476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076237.810524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913257088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076237.810558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076237.810572:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076237.810580:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076237.810603:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1943 00000100:00080000:1.0:1769076237.810621:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076237.810636:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076237.810659:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076237.810695:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913257216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076238.001209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076238.513211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076238.513265:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076238.513285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076238.833266:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076239.025203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076239.537235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076239.537249:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076239.537343:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076239.857238:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076240.049157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076240.561168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076240.561223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076240.561324:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076240.881234:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076241.073112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076241.585199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076241.585263:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076241.585290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076241.905220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076242.097156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076242.609162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076242.609170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076242.609235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076242.929199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076242.929214:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076242.929221:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076242.929228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076242.929235:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1948 00000100:00080000:1.0:1769076242.929254:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076242.929266:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076242.929321:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913257344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076242.929362:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076242.929367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076242.929370:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076242.929376:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1948 00000100:00080000:1.0:1769076242.929380:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076242.929384:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076242.929396:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076242.929410:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913257472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076242.929425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076242.929430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076242.929433:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076242.929439:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1948 00000100:00080000:1.0:1769076242.929444:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076242.929449:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076242.929460:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076242.929483:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913257600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076242.929499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076242.929504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076242.929506:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076242.929510:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1948 00000100:00080000:1.0:1769076242.929514:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076242.929519:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076242.929525:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076242.929540:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913257728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076242.929552:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076242.929557:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076242.929560:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076242.929566:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1948 00000100:00080000:1.0:1769076242.929578:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076242.929588:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076242.929598:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076242.929613:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913257856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076242.929635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076242.929640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076242.929643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076242.929648:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1948 00000100:00080000:1.0:1769076242.929653:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076242.929658:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076242.929664:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076242.929687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913257984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076242.929711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076242.929716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076242.929719:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076242.929723:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1948 00000100:00080000:1.0:1769076242.929727:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076242.929732:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076242.929738:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076242.929758:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913258112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076243.121137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076243.633249:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076243.633278:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076243.953247:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076244.145181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076244.657184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076244.657210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076244.657245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076244.977180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076245.169183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076245.681230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076245.681294:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076245.681341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076246.001251:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076246.193162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076246.705164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076246.705166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076246.705246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076247.025176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076247.217134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076247.729172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076247.729198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076247.729254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076248.049202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076248.049213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076248.049223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076248.049230:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076248.049243:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1953 00000100:00080000:2.0:1769076248.049259:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076248.049269:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076248.049321:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913258240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076248.049371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076248.049377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076248.049379:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076248.049385:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1953 00000100:00080000:2.0:1769076248.049388:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076248.049393:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076248.049406:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076248.049422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913258368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076248.049437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076248.049441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076248.049443:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076248.049449:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1953 00000100:00080000:2.0:1769076248.049454:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076248.049459:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076248.049465:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076248.049486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913258496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076248.049518:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076248.049523:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076248.049525:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076248.049529:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1953 00000100:00080000:2.0:1769076248.049535:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076248.049539:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076248.049544:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076248.049559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913258624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076248.049571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076248.049575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076248.049577:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076248.049583:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1953 00000100:00080000:2.0:1769076248.049588:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076248.049591:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076248.049596:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076248.049613:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913258752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076248.049623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076248.049633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076248.049637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076248.049649:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1953 00000100:00080000:2.0:1769076248.049657:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076248.049663:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076248.049670:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076248.049686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913258880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076248.049707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076248.049716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076248.049723:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076248.049761:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1953 00000100:00080000:2.0:1769076248.049773:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076248.049784:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076248.049794:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076248.049817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913259008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076248.241178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076248.753184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076248.753195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076248.753282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076249.073190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076249.265198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076249.777198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076249.777227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076249.777249:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076250.097211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076250.289181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076250.801230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076250.801323:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076250.801356:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076251.121130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076251.313177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076251.825180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076251.825219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076251.825284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076252.145242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076252.337167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076252.849150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076252.849247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076252.849352:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076253.169240:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076253.169269:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076253.169285:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076253.169292:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076253.169306:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1958 00000100:00080000:3.0:1769076253.169323:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076253.169333:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076253.169411:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913259136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076253.169455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076253.169460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076253.169463:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076253.169469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1958 00000100:00080000:3.0:1769076253.169473:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076253.169478:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076253.169493:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076253.169509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913259264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076253.169524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076253.169529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076253.169532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076253.169538:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1958 00000100:00080000:3.0:1769076253.169542:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076253.169547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076253.169553:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076253.169571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913259392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076253.169598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076253.169602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076253.169605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076253.169609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1958 00000100:00080000:3.0:1769076253.169614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076253.169618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076253.169639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076253.169690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913259520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076253.169703:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076253.169708:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076253.169711:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076253.169716:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1958 00000100:00080000:3.0:1769076253.169722:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076253.169728:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076253.169734:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076253.169751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913259648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076253.169768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076253.169772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076253.169775:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076253.169787:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1958 00000100:00080000:3.0:1769076253.169798:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076253.169808:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076253.169815:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076253.169836:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913259776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076253.169852:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076253.169856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076253.169859:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076253.169865:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1958 00000100:00080000:3.0:1769076253.169870:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076253.169882:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076253.169895:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076253.169916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913259904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076253.361193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076253.873231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076253.873231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076254.193200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076254.385180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076254.897229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076254.897238:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076254.897312:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076254.897324:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076255.217245:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076255.409142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076255.921214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076255.921260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076255.921330:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076256.241240:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076256.241496:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076256.241555:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076256.241585:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076256.241721:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076256.241740:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076256.241760:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076256.241782:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076256.241805:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076256.241830:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769076256.433136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076256.945298:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076256.945338:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076257.265169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076257.457162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076257.969221:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076257.969317:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076257.969337:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769076258.289158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076258.289177:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076258.289184:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076258.289200:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1963 00000100:00100000:1.0:1769076258.289243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076258.289253:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076258.289267:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076258.289317:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913260032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076258.289351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076258.289359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076258.289363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076258.289371:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1963 00000100:00080000:3.0:1769076258.289379:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076258.289386:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076258.289401:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076258.289430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913260160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076258.289450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076258.289456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076258.289459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076258.289466:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1963 00000100:00080000:3.0:1769076258.289473:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076258.289484:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076258.289512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076258.289539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913260288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076258.289613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076258.289622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076258.289628:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076258.289637:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1963 00000100:00080000:3.0:1769076258.289647:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076258.289656:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076258.289668:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076258.289702:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913260416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076258.289729:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076258.289736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076258.289740:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076258.289749:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1963 00000100:00080000:3.0:1769076258.289758:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076258.289769:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076258.289781:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076258.289815:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913260544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076258.289840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076258.289861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076258.289869:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076258.289883:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1963 00000100:00080000:3.0:1769076258.289898:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076258.289916:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076258.289935:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076258.289969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913260672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076258.289991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076258.289998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076258.290002:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076258.290026:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1963 00000100:00080000:3.0:1769076258.290044:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076258.290142:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076258.290159:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076258.290196:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913260800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076258.481171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076258.993224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076258.993225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076258.993282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076259.313316:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076259.505170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076260.017169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076260.017198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076260.017255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076260.337253:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076260.529156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076261.041161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076261.041226:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076261.361220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076261.553125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076262.065198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076262.065223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076262.385210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076262.577160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076263.089298:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076263.089328:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076263.409188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076263.409201:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1769076263.409202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076263.409206:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076263.409218:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1969 00000100:00080000:1.0:1769076263.409229:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076263.409235:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076263.409276:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913260928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076263.409299:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076263.409302:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076263.409304:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076263.409307:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1969 00000100:00080000:1.0:1769076263.409309:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076263.409312:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076263.409321:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076263.409331:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913261056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076263.409340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076263.409342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076263.409344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076263.409348:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1969 00000100:00080000:1.0:1769076263.409351:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076263.409354:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076263.409360:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076263.409368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913261184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076263.409375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076263.409377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076263.409379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076263.409381:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1969 00000100:00080000:1.0:1769076263.409383:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076263.409386:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076263.409389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076263.409397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913261312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076263.409403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076263.409405:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076263.409406:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076263.409409:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1969 00000100:00080000:1.0:1769076263.409419:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076263.409421:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076263.409424:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076263.409433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913261440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076263.409444:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076263.409447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076263.409448:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076263.409451:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1969 00000100:00080000:1.0:1769076263.409455:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076263.409457:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076263.409460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076263.409466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913261568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076263.409490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076263.409492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076263.409493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076263.409496:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1969 00000100:00080000:1.0:1769076263.409498:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076263.409501:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076263.409505:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076263.409514:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913261696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076263.409672:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913261056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076263.409680:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913261184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076263.409699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076263.409709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076263/real 1769076263] req@ffff9350100c9500 x1855008913261184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076263.409722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913261184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076263.409730:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076263.409733:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076263.409737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913261184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076263.409742:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913261312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076263.409758:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076263.409768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076263/real 1769076263] req@ffff9350100c9880 x1855008913261312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076263.409779:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913261312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076263.409787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076263.409790:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076263.409794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913261312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076263.409800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913261440:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076263.409816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076263.409825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076263/real 1769076263] req@ffff9350100c8e00 x1855008913261440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076263.409839:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913261440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076263.409847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076263.409850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076263.409854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913261440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076263.409860:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913261568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076263.409876:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076263.409886:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076263/real 1769076263] req@ffff9350100ca680 x1855008913261568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076263.409896:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913261568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076263.409903:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076263.409906:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076263.409909:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913261568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076263.409914:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913261696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076263.409929:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076263.409937:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076263/real 1769076263] req@ffff9350100ca300 x1855008913261696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076263.409948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913261696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076263.409962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076263.409965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076263.409969:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913261696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076263.601103:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076264.113218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076264.113231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076264.113251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076264.113344:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076264.433163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076264.625122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076265.137177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076265.137177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076265.137233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076265.137292:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076265.457134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076265.649152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076266.161173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076266.161179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076266.161211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076266.161248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076266.481220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076266.673237:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076267.185225:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076267.185238:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076267.185338:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076267.185346:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076267.505213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076267.697155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076268.209190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076268.209205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076268.209225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076268.209257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076268.529184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076268.529194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076268.529197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076268.529204:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076268.529216:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1974 00000100:00080000:3.0:1769076268.529229:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076268.529238:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076268.529294:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913261824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076268.529335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076268.529340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076268.529343:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076268.529350:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1974 00000100:00080000:3.0:1769076268.529355:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076268.529362:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076268.529376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076268.529446:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913261952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076268.529460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076268.529464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076268.529467:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076268.529488:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1974 00000100:00080000:3.0:1769076268.529496:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076268.529497:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913261824:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076268.529504:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076268.529520:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076268.529535:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913262080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076268.529554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076268.529558:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076268.529561:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076268.529582:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1974 00000100:00080000:3.0:1769076268.529586:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076268.529590:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076268.529595:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076268.529606:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913262208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076268.529617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076268.529621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076268.529623:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076268.529638:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1974 00000100:00080000:3.0:1769076268.529642:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076268.529645:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076268.529648:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076268.529657:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076268.529664:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913262336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076268.529678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076268.529683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076268.529685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076268.529690:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1974 00000100:00080000:3.0:1769076268.529714:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076268.529719:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076268.529728:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076268.529748:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913262464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076268.529756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076268/real 1769076268] req@ffff935002a7e300 x1855008913261824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076268.529764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076268.529775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076268.529777:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076268.529781:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1974 00000100:00080000:3.0:1769076268.529784:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076268.529789:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076268.529803:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076268.529831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913262592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076268.529848:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913261824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076268.529866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076268.529872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076268.529877:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913261824:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076268.529910:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913261952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076268.529942:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076268.529963:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076268/real 1769076268] req@ffff935002a7f100 x1855008913261952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076268.529983:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913261952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076268.530000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076268.530006:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076268.530011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913261952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076268.530022:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076268.530048:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076268.530059:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076268/real 1769076268] req@ffff935002a7c380 x1855008913262080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076268.530102:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913262080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076268.530111:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076268.530114:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076268.530118:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076268.530125:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076268.530143:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076268.530170:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076268/real 1769076268] req@ffff935002a7ca80 x1855008913262208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076268.530189:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913262208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076268.530202:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076268.530208:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076268.530214:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076268.530224:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913262336:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076268.530251:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076268.530263:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076268/real 1769076268] req@ffff935002a7fb80 x1855008913262336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076268.530297:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913262336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076268.530307:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076268.530311:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076268.530316:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913262336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076268.530323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913262464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076268.530344:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076268.530355:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076268/real 1769076268] req@ffff935002a7c700 x1855008913262464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076268.530369:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913262464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076268.530379:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076268.530382:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076268.530415:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913262464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076268.530433:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913262592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076268.530460:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076268.530475:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076268/real 1769076268] req@ffff935002a7d180 x1855008913262592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076268.530490:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913262592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076268.530500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076268.530504:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076268.530507:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913262592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076268.721210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076269.233215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076269.233285:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076269.233285:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076269.233326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076269.553163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076269.745119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076270.257181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076270.257181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076270.257217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076270.257278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076270.577202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076270.769230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076271.281184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076271.281218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076271.281297:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076271.281316:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076271.601218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076271.793144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076272.305231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076272.305234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076272.305305:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076272.305359:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076272.625207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076272.817205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076273.329160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076273.329170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076273.329213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076273.329225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076273.649265:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076273.649386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076273.649396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076273.649403:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076273.649426:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1979 00000100:00080000:0.0:1769076273.649441:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076273.649450:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076273.649504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913262720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076273.649549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076273.649554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076273.649592:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076273.649598:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1979 00000100:00080000:0.0:1769076273.649603:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076273.649609:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076273.649624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076273.649656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913262848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076273.649666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913262720:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076273.649685:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076273.649705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076273.649708:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076273.649716:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1979 00000100:00080000:0.0:1769076273.649721:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076273.649725:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076273.649733:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076273.649749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913262976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076273.649768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076273.649773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076273.649776:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076273.649780:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1979 00000100:00080000:0.0:1769076273.649786:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076273.649794:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076273.649802:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:0.0:1769076273.649805:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076273.649822:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913263104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076273.649844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076273.649849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076273.649851:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076273.649858:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1979 00000100:00080000:0.0:1769076273.649863:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076273.649873:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076273/real 1769076273] req@ffff9350103f9c00 x1855008913262720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076273.649884:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076273.649898:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076273.649935:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913263232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076273.649948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913262720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076273.649951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076273.649964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076273.649968:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076273.649992:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1979 00000100:00080000:0.0:1769076273.649998:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076273.650001:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076273.650004:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076273.650019:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076273.650025:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076273.650068:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913263360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076273.650071:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913262720:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076273.650166:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262848:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076273.650187:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076273.650195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076273.650199:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076273.650206:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1979 00000100:00080000:0.0:1769076273.650215:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076273.650222:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076273.650233:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:0.0:1769076273.650237:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076273.650259:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076273/real 1769076273] req@ffff9350103f9500 x1855008913262848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076273.650267:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913263488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076273.650363:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913262848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076273.650380:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076273.650385:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076273.650391:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076273.650401:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076273.650443:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076273.650458:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076273/real 1769076273] req@ffff9350103fad80 x1855008913262976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076273.650473:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913262976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076273.650482:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076273.650485:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076273.650489:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913262976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076273.650496:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913263104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076273.650513:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076273.650523:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076273/real 1769076273] req@ffff9350103fb800 x1855008913263104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076273.650535:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913263104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076273.650545:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076273.650549:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076273.650570:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913263104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076273.650577:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913263232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076273.650609:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076273.650619:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076273/real 1769076273] req@ffff9350103f9180 x1855008913263232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076273.650634:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913263232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076273.650656:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076273.650660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076273.650664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913263232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076273.650693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913263360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076273.650729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076273.650751:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076273/real 1769076273] req@ffff9350103f8700 x1855008913263360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076273.650776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913263360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076273.650794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076273.650800:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076273.650807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913263360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076273.650815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913263488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076273.650844:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076273.650859:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076273/real 1769076273] req@ffff9350103f8380 x1855008913263488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076273.650885:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913263488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076273.650895:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076273.650899:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076273.650904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913263488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076273.841207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076274.353190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076274.353234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076274.353345:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076274.353358:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076274.673306:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076274.865138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076275.377130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076275.377255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076275.377317:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076275.377342:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076275.697196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076275.889164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076276.401149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076276.401179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076276.401285:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076276.401307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076276.721317:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076276.913145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076277.425122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076277.425128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076277.425171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076277.425203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076277.745176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076277.937220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076278.449183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076278.449190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076278.449191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076278.449301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076278.769165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076278.769288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076278.769300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076278.769307:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076278.769320:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1984 00000100:00080000:3.0:1769076278.769337:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076278.769348:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076278.769401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913263616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076278.769442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076278.769447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076278.769450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076278.769456:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1984 00000100:00080000:3.0:1769076278.769461:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076278.769469:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076278.769487:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076278.769515:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913263616:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076278.769519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913263744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076278.769551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076278.769559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076278.769564:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076278.769572:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1984 00000100:00080000:3.0:1769076278.769582:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076278.769590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076278.769600:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076278.769620:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913263872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076278.769634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076278.769638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076278.769641:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076278.769652:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076278.769654:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1984 00000100:00080000:3.0:1769076278.769663:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076278.769668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076278.769676:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076278.769692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913264000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076278.769705:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076278.769710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076278.769713:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076278.769721:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1984 00000100:00100000:2.0:1769076278.769722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076278/real 1769076278] req@ffff935002a7dc00 x1855008913263616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076278.769726:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076278.769738:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076278.769752:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076278.769771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913263616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076278.769777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913264128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076278.769816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076278.769831:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076278.769845:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913263616:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076278.769853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076278.769863:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076278.769868:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076278.769878:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1984 00000100:00100000:2.0:1769076278.769886:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913263744:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076278.769888:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076278.769907:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076278.769920:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076278.769930:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076278.769951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913264256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076278.769954:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076278/real 1769076278] req@ffff935002a7f800 x1855008913263744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076278.770013:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913263744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076278.770016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076278.770036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076278.770042:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076278.770044:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076278.770091:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076278.770098:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913263744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076278.770109:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913263872:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076278.770127:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1984 00000100:00080000:3.0:1769076278.770140:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076278.770148:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076278.770157:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076278.770169:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076278.770170:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076278/real 1769076278] req@ffff935002a7c000 x1855008913263872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076278.770206:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913264384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076278.770207:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913263872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076278.770262:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076278.770269:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076278.770275:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913263872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076278.770285:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076278.770315:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076278.770330:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076278/real 1769076278] req@ffff935002a7df80 x1855008913264000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076278.770345:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913264000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076278.770355:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076278.770359:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076278.770363:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076278.770369:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913264128:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076278.770388:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076278.770399:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076278/real 1769076278] req@ffff935002a7d180 x1855008913264128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076278.770413:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913264128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076278.770423:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076278.770426:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076278.770430:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913264128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076278.770456:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913264256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076278.770474:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076278.770485:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076278/real 1769076278] req@ffff935002a7c700 x1855008913264256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076278.770499:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913264256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076278.770509:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076278.770513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076278.770519:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913264256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076278.770529:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913264384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076278.770563:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076278.770583:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076278/real 1769076278] req@ffff935002a7fb80 x1855008913264384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076278.770623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913264384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076278.770640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076278.770646:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076278.770651:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913264384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076278.961138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076279.473250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076279.473292:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076279.473308:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076279.473314:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076279.793230:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076279.985324:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076280.497251:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076280.497251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076280.497318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076280.497323:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076280.817267:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076281.009119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076281.521125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076281.521158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076281.521160:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076281.521206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076281.841265:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076282.033203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076282.545127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076282.545128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076282.545181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076282.545216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076282.865193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076283.057217:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076283.569206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076283.569243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076283.569277:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076283.569332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076283.889194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076283.889278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076283.889290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076283.889296:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076283.889308:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1989 00000100:00080000:0.0:1769076283.889324:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076283.889334:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076283.889385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913264512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076283.889429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076283.889434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076283.889437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076283.889444:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1989 00000100:00080000:0.0:1769076283.889449:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076283.889453:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076283.889466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076283.889482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913264640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076283.889498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076283.889502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076283.889504:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076283.889509:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1989 00000100:00080000:0.0:1769076283.889513:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076283.889519:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076283.889529:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913264512:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769076283.889531:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076283.889552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913264768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076283.889568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076283.889573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076283.889575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076283.889580:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1989 00000100:00080000:0.0:1769076283.889584:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076283.889589:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076283.889595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076283.889610:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913264896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076283.889625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076283.889629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076283.889632:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076283.889636:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1989 00000100:00080000:0.0:1769076283.889642:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076283.889644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076283.889653:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076283.889661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076283.889678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913265024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076283.889690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076283.889695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076283.889698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076283.889703:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1989 00000100:00080000:0.0:1769076283.889709:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076283.889715:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076283.889721:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076283.889726:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff9350103fb480 x1855008913264512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076283.889743:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913265152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076283.889773:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076283.889777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076283.889780:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076283.889785:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1989 00000100:00080000:0.0:1769076283.889796:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076283.889796:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913264512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076283.889812:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076283.889821:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076283.889841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076283.889844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913265280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076283.889859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076283.889876:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913264512:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076283.889890:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264640:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076283.889931:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076283.889950:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff9350103faa00 x1855008913264640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076283.889966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913264640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076283.889977:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076283.889981:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076283.889986:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076283.890017:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076283.890038:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076283.890096:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff9350103f9f80 x1855008913264768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076283.890113:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913264768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076283.890124:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076283.890154:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076283.890159:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076283.890168:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076283.890199:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076283.890214:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff9350103fbb80 x1855008913264896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076283.890230:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913264896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076283.890240:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076283.890244:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076283.890248:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913264896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076283.890254:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265024:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076283.890274:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076283.890284:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff9350103fb100 x1855008913265024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076283.890298:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913265024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076283.890308:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076283.890312:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076283.890315:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076283.890321:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076283.890339:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076283.890350:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff9350103f8380 x1855008913265152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076283.890363:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913265152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076283.890382:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076283.890386:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076283.890390:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076283.890396:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076283.890413:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076283.890423:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff9350103f8700 x1855008913265280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076283.890436:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913265280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076283.890445:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076283.890449:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076283.890453:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076284.081198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076284.593207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076284.593230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076284.593291:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076284.593307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076284.913206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076285.105218:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076285.617177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076285.617248:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076285.617292:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076285.617315:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076285.937288:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076286.129203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076286.641208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076286.641230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076286.641242:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076286.641335:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076286.961199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076287.153190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076287.665223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076287.665227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076287.665255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076287.665350:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076287.985185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076288.177263:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076288.689207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076288.689217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076288.689324:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076288.689357:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076289.009195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076289.009223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076289.009237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076289.009245:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076289.009258:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1994 00000100:00080000:1.0:1769076289.009275:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076289.009300:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076289.009365:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913265408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076289.009432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076289.009438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076289.009442:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076289.009448:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1994 00000100:00080000:1.0:1769076289.009455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076289.009461:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076289.009484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076289.009508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913265536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076289.009528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076289.009533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076289.009535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076289.009545:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1994 00000100:00080000:1.0:1769076289.009552:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076289.009567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076289.009568:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913265408:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769076289.009580:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076289.009600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913265664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076289.009615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076289.009621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076289.009632:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076289.009638:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1994 00000100:00080000:1.0:1769076289.009645:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076289.009651:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076289.009659:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076289.009673:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913265792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076289.009687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076289.009703:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076289.009707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076289.009711:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076289.009722:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1994 00000100:00080000:1.0:1769076289.009731:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076289.009738:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076289.009756:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076289.009784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913265920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076289.009795:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076289/real 1769076289] req@ffff9350100c8380 x1855008913265408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076289.009811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076289.009839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076289.009849:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076289.009869:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1994 00000100:00080000:1.0:1769076289.009877:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076289.009884:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913265408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076289.009890:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076289.009911:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076289.009921:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076289.009932:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913266048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076289.009933:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076289.009961:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913265408:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076289.009996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076289.010002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076289.010006:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076289.010015:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1994 00000100:00080000:1.0:1769076289.010019:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076289.010024:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076289.010036:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076289.010039:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913265536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076289.010157:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913266176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076289.010174:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076289.010215:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076289/real 1769076289] req@ffff9350100c9f80 x1855008913265536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076289.010239:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913265536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076289.010253:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076289.010258:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076289.010263:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913265536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076289.010273:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913265664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076289.010295:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076289.010307:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076289/real 1769076289] req@ffff9350100c9c00 x1855008913265664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076289.010321:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913265664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076289.010331:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076289.010336:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076289.010339:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913265664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076289.010348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913265792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076289.010366:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076289.010377:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076289/real 1769076289] req@ffff9350100c8a80 x1855008913265792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076289.010391:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913265792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076289.010401:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076289.010404:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076289.010408:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913265792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076289.010416:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265920:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076289.010449:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076289.010470:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076289/real 1769076289] req@ffff9350100c8000 x1855008913265920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076289.010494:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913265920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076289.010511:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076289.010518:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076289.010525:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913265920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076289.010539:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076289.010572:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076289.010587:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076289/real 1769076289] req@ffff9350100cb100 x1855008913266048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076289.010603:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913266048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076289.010613:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076289.010617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076289.010621:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076289.010648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076289.010668:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076289.010679:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076289/real 1769076289] req@ffff9350100c9180 x1855008913266176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076289.010693:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913266176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076289.010703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076289.010708:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076289.010711:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076289.201150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076289.713332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076289.713339:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076289.713347:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076289.713415:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076290.033284:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076290.225200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076290.737173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076290.737176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076290.737210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076290.737312:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076291.057182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076291.249136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076291.761142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076291.761142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076291.761203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076291.761208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076292.081138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076292.273137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076292.785156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076292.785198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076292.785223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076292.785326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076293.105181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076293.297175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076293.809233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076293.809257:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076293.809258:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076293.809281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076294.129201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076294.129217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1769076294.129219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076294.129224:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076294.129243:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 1999 00000100:00080000:3.0:1769076294.129263:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076294.129277:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076294.129367:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913266304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076294.129430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076294.129438:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076294.129442:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076294.129450:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1999 00000100:00080000:3.0:1769076294.129458:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076294.129467:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076294.129486:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076294.129512:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913266432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076294.129532:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076294.129538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076294.129541:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076294.129547:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1999 00000100:00080000:3.0:1769076294.129553:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076294.129564:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076294.129572:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076294.129576:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913266304:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076294.129588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913266560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076294.129607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076294.129611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076294.129613:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076294.129617:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 1999 00000100:00080000:3.0:1769076294.129626:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076294.129631:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076294.129637:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076294.129654:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913266688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076294.129666:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076294.129670:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076294.129672:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076294.129678:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1999 00000100:00080000:3.0:1769076294.129683:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076294.129687:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076294.129693:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076294.129701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076294.129720:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913266816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076294.129734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076294.129739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076294.129741:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076294.129746:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1999 00000100:00080000:3.0:1769076294.129752:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076294.129757:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076294.129763:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076294.129771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076294/real 1769076294] req@ffff935002a7ca80 x1855008913266304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076294.129785:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913266944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076294.129809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913266304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076294.129817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076294.129832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076294.129838:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076294.129846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076294.129848:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 1999 00000100:00080000:3.0:1769076294.129860:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076294.129865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076294.129872:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076294.129881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913266304:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076294.129882:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076294.129906:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913266432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076294.129912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913267072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076294.129948:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076294.129998:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076294/real 1769076294] req@ffff935002a7c380 x1855008913266432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076294.130015:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913266432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076294.130026:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076294.130031:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076294.130035:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913266432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076294.130117:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913266560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076294.130144:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076294.130158:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076294/real 1769076294] req@ffff935002a7f100 x1855008913266560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076294.130182:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913266560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076294.130192:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076294.130196:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076294.130200:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913266560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076294.130208:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913266688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076294.130226:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076294.130236:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076294/real 1769076294] req@ffff935002a7e300 x1855008913266688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076294.130250:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913266688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076294.130259:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076294.130263:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076294.130266:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913266688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076294.130272:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266816:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076294.130289:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076294.130299:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076294/real 1769076294] req@ffff935002a7fb80 x1855008913266816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076294.130313:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913266816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076294.130331:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076294.130335:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076294.130338:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076294.130344:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076294.130361:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076294.130371:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076294/real 1769076294] req@ffff935002a7c700 x1855008913266944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076294.130384:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913266944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076294.130393:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076294.130397:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076294.130401:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913266944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076294.130406:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076294.130423:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076294.130433:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076294/real 1769076294] req@ffff935002a7d180 x1855008913267072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076294.130446:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913267072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076294.130455:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076294.130459:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076294.130463:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076294.321187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076294.833214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076294.833218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076294.833249:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076294.833302:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076295.153190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076295.345146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076295.857175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076295.857196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076295.857198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076295.857236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076296.177219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076296.369251:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076296.881147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076296.881151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076296.881261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076296.881270:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076297.201139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076297.393142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076297.905126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076297.905139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076297.905169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076297.905177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076298.225159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076298.417172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076298.929244:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076298.929258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076298.929302:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076298.929314:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076299.249170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076299.249202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076299.249218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076299.249228:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076299.249261:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2004 00000100:00080000:3.0:1769076299.249280:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076299.249293:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076299.249355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913267200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076299.249403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076299.249410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076299.249414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076299.249423:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2004 00000100:00080000:3.0:1769076299.249429:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076299.249437:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076299.249454:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076299.249477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913267328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076299.249497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076299.249504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076299.249509:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076299.249517:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2004 00000100:00100000:2.0:1769076299.249526:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913267200:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076299.249527:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076299.249542:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076299.249554:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076299.249577:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913267456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076299.249598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076299.249604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076299.249608:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076299.249615:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2004 00000100:00080000:3.0:1769076299.249653:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076299.249662:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076299.249671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076299.249680:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076299.249710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913267584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076299.249746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076299.249753:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076299/real 1769076299] req@ffff935002a7df80 x1855008913267200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076299.249756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076299.249764:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076299.249781:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2004 00000100:00080000:3.0:1769076299.249791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076299.249795:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913267200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076299.249806:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076299.249817:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076299.249823:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076299.249826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076299.249836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913267200:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076299.249847:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913267328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076299.249860:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913267712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076299.249906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076299.249914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076299.249920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076299.249926:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076299.249936:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2004 00000100:00100000:2.0:1769076299.249940:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076299/real 1769076299] req@ffff935002a7c000 x1855008913267328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076299.249946:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076299.249964:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076299.249985:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913267328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076299.249998:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076299.250003:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076299.250012:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076299.250018:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913267328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076299.250035:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913267840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076299.250061:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913267456:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076299.250124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076299.250135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076299.250140:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076299.250152:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076299.250155:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2004 00000100:00080000:3.0:1769076299.250170:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076299.250183:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076299.250184:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076299/real 1769076299] req@ffff935002a7f800 x1855008913267456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076299.250205:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076299.250212:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913267456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076299.250227:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076299.250232:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076299.250238:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913267456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076299.250263:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913267968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076299.250270:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913267584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076299.250305:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076299.250319:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076299/real 1769076299] req@ffff935002a7dc00 x1855008913267584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076299.250334:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913267584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076299.250343:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076299.250346:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076299.250349:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913267584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076299.250354:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267712:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076299.250371:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076299.250380:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076299/real 1769076299] req@ffff935002a7d180 x1855008913267712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076299.250393:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913267712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076299.250401:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076299.250404:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076299.250407:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076299.250426:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076299.250442:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076299.250452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076299/real 1769076299] req@ffff935002a7c700 x1855008913267840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076299.250464:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913267840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076299.250473:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076299.250476:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076299.250479:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076299.250486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076299.250512:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076299.250530:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076299/real 1769076299] req@ffff935002a7fb80 x1855008913267968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076299.250563:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913267968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076299.250578:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076299.250583:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076299.250588:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913267968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076299.441168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076299.953148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076299.953177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076299.953190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076299.953271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076300.273209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076300.465168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076300.977169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076300.977178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076300.977257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076300.977276:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076301.297176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076301.489141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076302.001217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076302.001219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076302.001238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076302.001294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076302.321139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076302.513177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076303.025196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076303.025202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076303.025248:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076303.025254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076303.345162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076303.537146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076304.049177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076304.049196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076304.049204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076304.049269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076304.369169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076304.369182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076304.369199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076304.369207:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076304.369223:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2010 00000100:00080000:3.0:1769076304.369243:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076304.369258:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076304.369316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913268096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076304.369359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076304.369365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076304.369367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076304.369374:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2010 00000100:00080000:3.0:1769076304.369379:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076304.369384:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076304.369397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076304.369412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913268224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076304.369426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076304.369431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076304.369433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076304.369438:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2010 00000100:00080000:3.0:1769076304.369442:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076304.369448:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076304.369453:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076304.369465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913268352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076304.369479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076304.369487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076304.369492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076304.369493:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913268096:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076304.369500:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2010 00000100:00080000:3.0:1769076304.369509:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076304.369517:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076304.369523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076304.369538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913268480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076304.369549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076304.369554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076304.369556:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076304.369574:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2010 00000100:00080000:3.0:1769076304.369595:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076304.369601:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076304.369608:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076304.369614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076304.369633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913268608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076304.369650:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076304.369654:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076304.369657:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076304.369662:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2010 00000100:00080000:3.0:1769076304.369668:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076304.369674:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076304.369680:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076304.369682:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076304/real 1769076304] req@ffff935002a7e300 x1855008913268096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076304.369702:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913268736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076304.369718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913268096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076304.369720:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076304.369736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076304.369743:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076304.369757:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2010 00000100:00080000:2.0:1769076304.369761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076304.369771:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076304.369772:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076304.369782:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913268096:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076304.369812:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076304.369821:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076304.369836:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913268224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076304.369847:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913268864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076304.369887:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076304.369907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076304/real 1769076304] req@ffff935002a7f100 x1855008913268224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076304.369934:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913268224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076304.369946:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076304.369950:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076304.369955:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913268224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076304.369990:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913268352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076304.370012:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076304.370024:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076304/real 1769076304] req@ffff935002a7c380 x1855008913268352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076304.370039:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913268352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076304.370091:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076304.370096:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076304.370100:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913268352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076304.370109:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913268480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076304.370132:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076304.370145:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076304/real 1769076304] req@ffff935002a7ca80 x1855008913268480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076304.370160:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913268480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076304.370178:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076304.370182:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076304.370186:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913268480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076304.370193:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913268608:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076304.370210:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076304.370221:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076304/real 1769076304] req@ffff935002a7fb80 x1855008913268608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076304.370234:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913268608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076304.370245:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076304.370248:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076304.370252:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913268608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076304.370259:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913268736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076304.370277:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076304.370287:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076304/real 1769076304] req@ffff935002a7c700 x1855008913268736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076304.370301:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913268736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076304.370311:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076304.370315:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076304.370319:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913268736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076304.370326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913268864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076304.370343:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076304.370354:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076304/real 1769076304] req@ffff935002a7d180 x1855008913268864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076304.370367:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913268864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076304.370383:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076304.370388:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076304.370396:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913268864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076304.561151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076305.073187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076305.073195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076305.073203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076305.073268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076305.393175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076305.585206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076306.097240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076306.097301:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076306.097315:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076306.097430:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076306.417244:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076306.609194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076307.121196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076307.121202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076307.121208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076307.121257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076307.441179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076307.633145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076308.145231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076308.145292:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076308.145295:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076308.145342:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076308.465199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076308.657202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076309.169248:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076309.169250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076309.169263:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076309.169318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076309.489118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076309.489124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076309.489130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076309.489135:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076309.489144:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2015 00000100:00080000:3.0:1769076309.489174:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076309.489184:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076309.489237:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913268992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076309.489273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076309.489278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076309.489280:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076309.489286:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2015 00000100:00080000:3.0:1769076309.489290:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076309.489294:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076309.489306:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076309.489322:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913269120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076309.489337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076309.489342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076309.489345:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076309.489350:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2015 00000100:00080000:3.0:1769076309.489354:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076309.489359:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076309.489367:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076309.489372:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913268992:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076309.489384:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913269248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076309.489409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076309.489413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076309.489415:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076309.489420:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2015 00000100:00080000:3.0:1769076309.489424:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076309.489428:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076309.489434:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076309.489449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913269376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076309.489460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076309.489465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076309.489467:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076309.489471:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2015 00000100:00080000:3.0:1769076309.489476:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076309.489485:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076309.489490:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076309.489499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076309.489516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913269504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076309.489528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076309.489533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076309.489535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076309.489539:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2015 00000100:00080000:3.0:1769076309.489545:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076309.489551:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076309.489559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076309.489568:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076309/real 1769076309] req@ffff935002a7dc00 x1855008913268992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076309.489577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913269632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076309.489614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076309.489619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076309.489622:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076309.489629:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913268992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076309.489633:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2015 00000100:00080000:3.0:1769076309.489660:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076309.489666:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076309.489670:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076309.489678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076309.489679:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913269760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076309.489693:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076309.489716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913268992:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076309.489723:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913269120:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076309.489743:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076309.489752:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076309/real 1769076309] req@ffff935002a7f800 x1855008913269120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076309.489761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913269120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076309.489767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076309.489769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076309.489772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913269120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076309.489794:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913269248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076309.489806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076309.489812:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076309/real 1769076309] req@ffff935002a7c000 x1855008913269248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076309.489820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913269248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076309.489825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076309.489828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076309.489830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913269248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076309.489833:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913269376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076309.489842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076309.489847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076309/real 1769076309] req@ffff935002a7df80 x1855008913269376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076309.489859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913269376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076309.489865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076309.489866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076309.489868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913269376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076309.489872:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913269504:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076309.489881:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076309.489887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076309/real 1769076309] req@ffff935002a7d180 x1855008913269504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076309.489895:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913269504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076309.489900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076309.489902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076309.489904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913269504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076309.489907:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913269632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076309.489917:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076309.489922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076309/real 1769076309] req@ffff935002a7c700 x1855008913269632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076309.489930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913269632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076309.489963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076309.489967:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076309.489971:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913269632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076309.489978:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913269760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076309.489998:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076309.490008:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076309/real 1769076309] req@ffff935002a7fb80 x1855008913269760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076309.490022:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913269760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076309.490031:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076309.490035:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076309.490038:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913269760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076309.681203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076310.193139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076310.193139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076310.193186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076310.193205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076310.513150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076310.705168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076311.217186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076311.217207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076311.217235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076311.217336:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076311.537180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076311.729184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076312.241195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076312.241212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076312.241221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076312.241299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076312.561196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076312.753169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076313.265199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076313.265205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076313.265249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076313.265284:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076313.585136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076313.777177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076314.289195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076314.289198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076314.289299:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076314.289326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076314.609183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076314.609221:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076314.609233:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076314.609240:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076314.609256:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2020 00000100:00080000:0.0:1769076314.609276:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076314.609289:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076314.609342:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913269888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076314.609385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076314.609393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076314.609397:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076314.609405:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2020 00000100:00080000:0.0:1769076314.609413:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076314.609421:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076314.609437:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076314.609461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913270016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076314.609463:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913269888:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076314.609482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076314.609488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076314.609492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076314.609498:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2020 00000100:00080000:0.0:1769076314.609504:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076314.609510:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076314.609519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076314.609536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913270144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076314.609551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076314.609558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076314.609561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076314.609568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2020 00000800:00000100:2.0:1769076314.609569:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076314.609573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076314.609580:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076314.609587:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076314.609605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913270272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076314.609658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076314.609664:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076314.609668:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076314.609676:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2020 00000100:00100000:2.0:1769076314.609680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076314/real 1769076314] req@ffff9350103f9180 x1855008913269888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076314.609683:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076314.609691:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076314.609701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076314.609716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913269888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076314.609719:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913270400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076314.609738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076314.609739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076314.609745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076314.609748:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076314.609749:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076314.609755:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2020 00000100:00100000:2.0:1769076314.609757:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913269888:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076314.609761:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076314.609768:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076314.609776:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076314.609792:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913270528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076314.609797:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270016:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076314.609811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076314.609817:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076314.609821:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076314.609828:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2020 00000100:00080000:0.0:1769076314.609834:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076314.609839:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076314.609841:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076314.609863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076314/real 1769076314] req@ffff9350103fb800 x1855008913270016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076314.609869:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076314.609892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913270656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076314.609892:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913270016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076314.609912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076314.609918:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076314.609925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076314.609937:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076314.609977:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076314.610000:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076314/real 1769076314] req@ffff9350103fad80 x1855008913270144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076314.610017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913270144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076314.610029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076314.610033:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076314.610037:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076314.610046:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076314.610122:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076314.610138:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076314/real 1769076314] req@ffff9350103f9500 x1855008913270272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076314.610153:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913270272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076314.610163:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076314.610168:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076314.610172:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076314.610179:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913270400:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076314.610198:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076314.610209:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076314/real 1769076314] req@ffff9350103f9c00 x1855008913270400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076314.610223:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913270400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076314.610233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076314.610236:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076314.610240:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913270400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076314.610264:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913270528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076314.610284:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076314.610296:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076314/real 1769076314] req@ffff9350103f9180 x1855008913270528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076314.610311:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913270528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076314.610321:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076314.610324:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076314.610328:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913270528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076314.610334:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913270656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076314.610352:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076314.610362:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076314/real 1769076314] req@ffff9350103f8700 x1855008913270656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076314.610376:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913270656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076314.610386:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076314.610389:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076314.610393:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913270656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076314.801159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076315.313233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076315.313255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076315.313300:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076315.313301:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076315.633195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076315.825196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076316.337209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076316.337221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076316.337256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076316.337263:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076316.657190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076316.849197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076317.361152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076317.361199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076317.361276:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076317.361281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076317.681229:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076317.873196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076318.385242:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076318.385246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076318.385327:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076318.385349:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076318.705221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076318.897175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076319.409170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076319.409199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076319.409288:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076319.409309:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076319.729177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076319.729209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076319.729227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076319.729234:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076319.729248:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2025 00000100:00080000:0.0:1769076319.729266:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076319.729276:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076319.729347:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913270784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076319.729393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076319.729400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076319.729403:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076319.729408:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2025 00000100:00080000:0.0:1769076319.729413:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076319.729418:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076319.729432:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076319.729451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913270912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076319.729465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076319.729469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076319.729471:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076319.729477:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2025 00000100:00080000:0.0:1769076319.729480:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076319.729484:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076319.729490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076319.729501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913271040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076319.729514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076319.729519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076319.729522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076319.729563:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2025 00000100:00080000:0.0:1769076319.729569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076319.729575:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076319.729583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076319.729597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913270784:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076319.729606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913271168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076319.729625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076319.729630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076319.729633:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076319.729639:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2025 00000100:00080000:0.0:1769076319.729644:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076319.729648:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076319.729656:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076319.729672:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913271296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076319.729684:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076319.729690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076319.729692:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076319.729696:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2025 00000100:00080000:0.0:1769076319.729702:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076319.729706:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076319.729712:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076319.729725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913271424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076319.729737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076319.729742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076319.729745:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076319.729752:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2025 00000100:00080000:0.0:1769076319.729758:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076319.729765:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270912:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076319.729771:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076319.729787:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076319.729808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913271552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076319.729852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076319.729927:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076319.729969:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913271296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076319.729997:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913271424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076319.730034:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913271552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076319.921195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076320.433188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076320.433199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076320.433204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076320.433302:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076320.753294:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076320.945181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076321.457253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076321.457298:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076321.457326:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076321.457366:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076321.777189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076321.969171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076322.481230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076322.481252:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076322.481252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076322.481403:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076322.801515:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076322.801681:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076322.801737:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076322.801762:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076322.801831:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076322.801848:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076322.801876:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076322.801901:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076322.801916:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076322.801931:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076322.802038:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076319/real 1769076322] req@ffff9350103f9f80 x1855008913271168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076322.802131:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913271168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076322.802146:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076322.802153:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076322.802158:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076322.802169:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076319/real 1769076322] req@ffff9350103faa00 x1855008913271296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076322.802186:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913271296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076322.802196:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076322.802201:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076322.802206:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913271296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076322.802216:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076319/real 1769076322] req@ffff9350103fb480 x1855008913271424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076322.802234:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913271424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076322.802251:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076322.802257:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076322.802261:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913271424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076322.802272:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076319/real 1769076322] req@ffff9350103f8700 x1855008913271552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076322.802286:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913271552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076322.802296:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076322.802300:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076322.802305:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913271552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076322.802315:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076319/real 1769076322] req@ffff9350103f8380 x1855008913270784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076322.802332:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913270784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076322.802341:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076322.802345:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076322.802351:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913270784:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076322.802361:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076319/real 1769076322] req@ffff9350103fb100 x1855008913270912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076322.802377:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913270912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076322.802388:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076322.802393:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076322.802397:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913270912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076322.802407:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076319/real 1769076322] req@ffff9350103fbb80 x1855008913271040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076322.802420:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913271040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076322.802428:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076322.802432:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076322.802437:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076322.993166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076323.505194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076323.505194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076323.505198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076323.505284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076323.825284:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076324.017173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076324.529165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076324.529172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076324.529190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076324.529243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076324.849337:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076324.849351:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076324.849367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076324.849374:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076324.849390:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2030 00000100:00080000:3.0:1769076324.849418:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076324.849429:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076324.849525:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913271680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076324.849573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076324.849578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076324.849581:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076324.849598:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2030 00000100:00080000:3.0:1769076324.849602:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076324.849606:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076324.849619:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076324.849634:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913271808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076324.849647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076324.849652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076324.849655:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076324.849661:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2030 00000100:00080000:3.0:1769076324.849664:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076324.849669:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076324.849675:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076324.849699:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913271936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076324.849704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913271680:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076324.849732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076324.849740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076324.849745:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076324.849756:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2030 00000100:00080000:3.0:1769076324.849764:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076324.849773:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076324.849784:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076324.849808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913272064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076324.849820:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076324.849841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076324.849849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076324.849854:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076324.849865:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2030 00000100:00080000:3.0:1769076324.849873:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076324.849883:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076324.849894:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076324.849907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076324/real 1769076324] req@ffff935002a7ca80 x1855008913271680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076324.849936:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913272192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076324.849954:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913271680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076324.849968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076324.849990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076324.849998:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076324.850002:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076324.850011:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2030 00000100:00080000:0.0:1769076324.850021:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076324.850024:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076324.850036:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913271680:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076324.850044:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076324.850102:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271808:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769076324.850147:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076324.850160:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076324.850182:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913272320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076324.850200:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076324/real 1769076324] req@ffff935002a7c380 x1855008913271808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076324.850258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076324.850267:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913271808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076324.850271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076324.850280:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076324.850302:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2030 00000100:00080000:0.0:1769076324.850307:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076324.850313:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076324.850323:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076324.850331:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076324.850341:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076324.850392:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076324.850465:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076324.850496:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076324/real 1769076324] req@ffff935002a7f100 x1855008913271936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.850524:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913271936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.850542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.850551:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076324.850560:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913271936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076324.850579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272064:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769076324.850581:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076324.850623:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913272448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076324.850640:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076324.850683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076324/real 1769076324] req@ffff935002a7e300 x1855008913272064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.850711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913272064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.850752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.850760:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076324.850766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076324.850778:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913272192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076324.850832:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076324.850859:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076324/real 1769076324] req@ffff935002a7fb80 x1855008913272192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.850884:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913272192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.850900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.850906:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076324.850912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913272192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076324.850924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913272320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076324.850962:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076324.850986:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076324/real 1769076324] req@ffff935002a7c700 x1855008913272320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.851011:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913272320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.851028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.851034:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076324.851041:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913272320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076324.851132:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913272448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076324.851174:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076324.851199:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076324/real 1769076324] req@ffff935002a7d180 x1855008913272448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.851224:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913272448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.851239:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.851245:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076324.851252:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913272448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076325.041152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076325.553195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076325.553196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076325.553209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076325.553277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076325.873380:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076326.065153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076326.577193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076326.577228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076326.577279:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076326.577297:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076326.897149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076327.089199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076327.601210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076327.601214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076327.601268:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076327.601316:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076327.921255:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076328.113210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076328.625186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076328.625202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076328.625263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076328.625317:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076328.945145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076329.137187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076329.649170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076329.649231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076329.649279:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076329.649281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076329.969202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076329.969255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076329.969266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076329.969273:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076329.969284:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2035 00000100:00080000:0.0:1769076329.969300:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076329.969310:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076329.969422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913272576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076329.969491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076329.969498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076329.969503:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076329.969512:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2035 00000100:00080000:0.0:1769076329.969520:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076329.969529:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076329.969547:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076329.969579:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913272704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076329.969595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076329.969600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076329.969603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076329.969609:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2035 00000100:00080000:0.0:1769076329.969614:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076329.969621:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076329.969636:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076329.969639:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913272576:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076329.969657:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913272832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076329.969675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076329.969679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076329.969681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076329.969688:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2035 00000100:00080000:0.0:1769076329.969693:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076329.969699:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076329.969724:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076329.969759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913272960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076329.969770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076329.969774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076329.969777:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076329.969781:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2035 00000100:00080000:0.0:1769076329.969790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076329.969800:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076329.969802:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076329.969821:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076329.969837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913273088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076329.969851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076329.969855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076329.969857:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076329.969863:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2035 00000100:00080000:0.0:1769076329.969867:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076329.969872:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076329.969878:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076329.969906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913273216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076329.969908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9350103fbb80 x1855008913272576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076329.969997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076329.970008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076329.970012:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913272576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076329.970016:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076329.970044:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2035 00000100:00080000:0.0:1769076329.970074:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076329.970140:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076329.970153:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076329.970155:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076329.970172:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076329.970195:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913272576:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076329.970202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913273344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076329.970236:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272704:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076329.970293:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076329.970318:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9350103fb100 x1855008913272704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076329.970379:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913272704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076329.970397:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076329.970404:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076329.970411:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076329.970472:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076329.970512:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076329.970534:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9350103f8380 x1855008913272832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076329.970558:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913272832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076329.970574:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076329.970581:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076329.970587:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076329.970599:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076329.970635:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076329.970655:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9350103f8700 x1855008913272960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076329.970678:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913272960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076329.970693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076329.970710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076329.970717:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913272960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076329.970729:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273088:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076329.970764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076329.970784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9350103fb480 x1855008913273088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076329.970806:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913273088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076329.970821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076329.970828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076329.970834:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076329.970845:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076329.970877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076329.970897:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9350103faa00 x1855008913273216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076329.970932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913273216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076329.970960:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076329.970966:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076329.970973:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076329.970984:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076329.971020:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076329.971041:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9350103f9f80 x1855008913273344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076329.971120:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913273344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076329.971138:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076329.971145:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076329.971152:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076330.161152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076330.673178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076330.673203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076330.673264:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076330.673373:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076330.993154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076331.185150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076331.697229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076331.697293:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076331.697311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076331.697316:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076332.017165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076332.209191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076332.721179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076332.721233:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076332.721262:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076332.721394:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076333.041174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076333.233171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076333.745205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076333.745218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076333.745219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076333.745372:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076334.065159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076334.257218:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076334.769194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076334.769224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076334.769309:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076334.769345:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076335.089212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076335.089361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076335.089375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076335.089381:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076335.089393:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2040 00000100:00080000:3.0:1769076335.089410:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076335.089420:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076335.089471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913273472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076335.089514:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076335.089519:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076335.089522:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076335.089528:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2040 00000100:00080000:3.0:1769076335.089532:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076335.089537:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076335.089559:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076335.089576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913273600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076335.089590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076335.089595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076335.089597:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076335.089603:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2040 00000100:00080000:3.0:1769076335.089607:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076335.089613:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076335.089621:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076335.089625:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913273472:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076335.089636:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913273728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076335.089651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076335.089656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076335.089660:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076335.089666:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2040 00000100:00080000:3.0:1769076335.089670:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076335.089675:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076335.089682:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076335.089702:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913273856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076335.089720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076335.089731:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076335.089740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076335.089746:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076335.089760:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2040 00000100:00080000:3.0:1769076335.089770:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076335.089782:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076335.089792:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076335/real 1769076335] req@ffff935002a7df80 x1855008913273472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076335.089795:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076335.089855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913273984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076335.089862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913273472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076335.089877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076335.089898:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076335.089904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076335.089912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913273472:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076335.089916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076335.089925:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076335.089935:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913273600:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076335.089938:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2040 00000100:00080000:3.0:1769076335.089958:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076335.089970:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076335.089978:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076335.089987:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076335.090004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913274112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076335.090013:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076335/real 1769076335] req@ffff935002a7c000 x1855008913273600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076335.090096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076335.090102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076335.090105:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076335.090129:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2040 00000100:00080000:3.0:1769076335.090135:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076335.090139:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913273600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076335.090151:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076335.090177:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076335.090180:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076335.090190:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076335.090200:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913273600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076335.090205:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913274240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076335.090259:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913273728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076335.090318:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076335.090328:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076335/real 1769076335] req@ffff935002a7f800 x1855008913273728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076335.090339:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913273728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076335.090351:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076335.090354:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076335.090357:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913273728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076335.090362:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913273856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076335.090374:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076335.090381:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076335/real 1769076335] req@ffff935002a7dc00 x1855008913273856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076335.090389:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913273856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076335.090396:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076335.090398:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076335.090400:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913273856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076335.090404:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273984:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076335.090416:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076335.090422:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076335/real 1769076335] req@ffff935002a7d180 x1855008913273984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076335.090431:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913273984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076335.090437:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076335.090440:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076335.090443:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913273984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076335.090448:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913274112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076335.090457:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076335.090463:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076335/real 1769076335] req@ffff935002a7c700 x1855008913274112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076335.090471:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913274112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076335.090477:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076335.090479:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076335.090482:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913274112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076335.090486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913274240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076335.090496:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076335.090503:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076335/real 1769076335] req@ffff935002a7fb80 x1855008913274240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076335.090510:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913274240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076335.090516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076335.090519:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076335.090521:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913274240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076335.281177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076335.793187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076335.793193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076335.793217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076335.793307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076336.113153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076336.305121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076336.817147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076336.817161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076336.817161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076336.817392:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076337.137157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076337.329161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076337.841262:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076337.841270:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076337.841285:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076337.841354:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076338.161202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076338.353180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076338.865164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076338.865173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076338.865177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076338.865389:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076339.185171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076339.377175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076339.889221:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076339.889276:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076339.889281:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076339.889382:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076340.209222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076340.209222:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076340.209239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076340.209245:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076340.209257:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2045 00000100:00080000:1.0:1769076340.209274:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076340.209285:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076340.209336:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913274368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076340.209374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076340.209379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076340.209381:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076340.209386:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2045 00000100:00080000:1.0:1769076340.209390:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076340.209395:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076340.209408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076340.209438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913274496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076340.209454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076340.209460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076340.209465:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076340.209472:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913274368:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076340.209474:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2045 00000100:00080000:1.0:1769076340.209488:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076340.209496:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076340.209503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076340.209521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913274624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076340.209535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076340.209540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076340.209543:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076340.209548:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2045 00000100:00080000:1.0:1769076340.209555:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076340.209559:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076340.209565:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076340.209579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913274752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076340.209590:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076340.209623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076340.209629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076340.209631:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076340.209640:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2045 00000100:00080000:1.0:1769076340.209646:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076340.209654:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076340.209661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076340.209665:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076340/real 1769076340] req@ffff9350100cad80 x1855008913274368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076340.209682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913274880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076340.209697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076340.209700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913274368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076340.209711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076340.209720:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076340.209740:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2045 00000100:00080000:1.0:1769076340.209756:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076340.209762:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076340.209765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769076340.209771:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076340.209777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076340.209811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913274368:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076340.209812:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913275008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076340.209846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076340.209851:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076340.209854:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076340.209860:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2045 00000100:00080000:1.0:1769076340.209867:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076340.209875:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076340.209881:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913274496:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769076340.209888:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076340.209918:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913275136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076340.209951:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076340.209981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076340/real 1769076340] req@ffff9350100c9180 x1855008913274496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076340.210016:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913274496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076340.210026:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076340.210030:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076340.210034:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913274496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076340.210041:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913274624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076340.210076:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076340.210085:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076340/real 1769076340] req@ffff9350100cb100 x1855008913274624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076340.210104:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913274624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076340.210111:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076340.210114:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076340.210116:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913274624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076340.210122:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913274752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076340.210278:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076340.210297:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076340/real 1769076340] req@ffff9350100c8000 x1855008913274752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076340.210316:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913274752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076340.210327:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076340.210330:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076340.210334:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913274752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076340.210340:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913274880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076340.210362:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076340.210372:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076340/real 1769076340] req@ffff9350100c8a80 x1855008913274880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076340.210382:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913274880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076340.210396:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076340.210400:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076340.210403:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913274880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076340.210408:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076340.210421:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076340.210429:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076340/real 1769076340] req@ffff9350100c9c00 x1855008913275008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076340.210438:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913275008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076340.210446:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076340.210449:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076340.210451:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076340.210471:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076340.210485:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076340.210493:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076340/real 1769076340] req@ffff9350100c9f80 x1855008913275136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076340.210504:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913275136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076340.210511:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076340.210514:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076340.210518:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076340.401160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076340.913240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076340.913250:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076340.913342:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076340.913435:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076341.233292:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076341.425119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076341.937240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076341.937278:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076341.937307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076341.937314:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076342.257162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076342.449245:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076342.961232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076342.961242:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076342.961271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076342.961301:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076343.281251:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076343.473208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076343.985214:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076343.985215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076343.985215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076343.985274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076344.305198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076344.497193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076345.009243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076345.009244:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076345.009246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076345.009300:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076345.329258:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076345.329281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076345.329292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076345.329298:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076345.329314:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2051 00000100:00080000:0.0:1769076345.329331:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076345.329341:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076345.329413:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913275264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076345.329453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076345.329458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076345.329461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076345.329466:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2051 00000100:00080000:0.0:1769076345.329471:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076345.329476:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076345.329488:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076345.329503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913275392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076345.329516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076345.329520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076345.329522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076345.329527:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2051 00000100:00080000:0.0:1769076345.329530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076345.329534:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076345.329543:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076345.329563:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913275520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076345.329583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076345.329591:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076345.329596:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913275264:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076345.329598:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076345.329616:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2051 00000100:00080000:0.0:1769076345.329624:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076345.329632:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076345.329643:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076345.329667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913275648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076345.329702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076345.329707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076345.329709:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076345.329714:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2051 00000100:00080000:0.0:1769076345.329721:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076345.329731:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076345.329733:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076345.329746:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076345.329764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913275776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076345.329777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076345.329781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076345.329783:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076345.329788:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2051 00000100:00080000:0.0:1769076345.329794:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076345.329798:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076345.329805:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076345.329811:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076345/real 1769076345] req@ffff9350103f9180 x1855008913275264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076345.329827:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913275904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076345.329844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076345.329851:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076345.329853:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913275264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076345.329855:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076345.329869:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2051 00000100:00080000:0.0:1769076345.329878:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076345.329890:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076345.329896:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769076345.329898:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076345.329913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076345.329921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913276032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076345.329927:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913275264:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076345.329953:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913275392:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076345.330003:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076345.330031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076345/real 1769076345] req@ffff9350103f9c00 x1855008913275392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076345.330141:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913275392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076345.330155:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076345.330160:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076345.330165:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913275392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076345.330202:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913275520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076345.330233:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076345.330248:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076345/real 1769076345] req@ffff9350103f9500 x1855008913275520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076345.330263:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913275520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076345.330274:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076345.330278:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076345.330283:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913275520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076345.330289:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913275648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076345.330308:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076345.330318:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076345/real 1769076345] req@ffff9350103fad80 x1855008913275648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076345.330331:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913275648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076345.330342:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076345.330345:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076345.330349:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913275648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076345.330356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275776:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076345.330374:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076345.330385:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076345/real 1769076345] req@ffff9350103fb800 x1855008913275776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076345.330399:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913275776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076345.330409:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076345.330412:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076345.330416:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076345.330422:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076345.330439:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076345.330449:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076345/real 1769076345] req@ffff9350103f9f80 x1855008913275904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076345.330463:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913275904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076345.330473:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076345.330476:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076345.330480:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913275904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076345.330486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076345.330503:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076345.330513:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076345/real 1769076345] req@ffff9350103faa00 x1855008913276032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076345.330525:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913276032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076345.330543:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076345.330547:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076345.330551:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076345.521158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076346.033149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076346.033237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076346.033291:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076346.033294:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076346.353248:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076346.545215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076347.057206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076347.057226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076347.057257:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076347.057309:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076347.377218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076347.569173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076348.081211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076348.081212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076348.081223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076348.081271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076348.401210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076348.593199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076349.105192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076349.105246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076349.105256:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076349.105344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076349.425187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076349.617164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076350.129204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076350.129230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076350.129282:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076350.129386:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076350.449178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076350.449204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076350.449216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076350.449222:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076350.449235:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2056 00000100:00080000:0.0:1769076350.449248:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076350.449255:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076350.449327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913276160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076350.449363:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076350.449369:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076350.449371:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076350.449376:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2056 00000100:00080000:0.0:1769076350.449389:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076350.449398:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076350.449413:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076350.449425:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913276160:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076350.449432:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913276288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076350.449448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076350.449453:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076350.449455:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076350.449460:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2056 00000100:00080000:0.0:1769076350.449465:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076350.449470:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076350.449476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076350.449491:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913276416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076350.449500:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076350.449510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076350.449515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076350.449518:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076350.449525:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2056 00000100:00080000:0.0:1769076350.449530:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076350.449535:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076350.449544:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076350.449552:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076350/real 1769076350] req@ffff9350103fb480 x1855008913276160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076350.449560:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913276544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076350.449575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076350.449576:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913276160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076350.449587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076350.449592:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076350.449596:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076350.449602:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076350.449606:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913276160:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076350.449613:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2056 00000100:00080000:0.0:1769076350.449619:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076350.449625:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076350.449632:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076350.449638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913276288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076350.449648:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913276672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076350.449660:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076350.449681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076350.449686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076350.449689:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076350.449692:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076350/real 1769076350] req@ffff9350103f8700 x1855008913276288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076350.449697:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2056 00000100:00080000:0.0:1769076350.449705:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076350.449710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913276288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076350.449712:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076350.449721:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076350.449724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076350.449730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076350.449732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913276288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076350.449740:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913276800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076350.449740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913276416:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769076350.449758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076350.449762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769076350.449763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076350.449767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076350.449774:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2056 00000100:00100000:2.0:1769076350.449775:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076350/real 1769076350] req@ffff9350103f8380 x1855008913276416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076350.449781:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076350.449789:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913276416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076350.449790:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076350.449798:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076350.449800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076350.449805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076350.449808:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913276416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076350.449813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913276928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076350.449816:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913276544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076350.449838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076350.449845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076350/real 1769076350] req@ffff9350103fb100 x1855008913276544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076350.449855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913276544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076350.449861:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076350.449863:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076350.449866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913276544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076350.449876:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276672:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076350.449886:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076350.449892:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076350/real 1769076350] req@ffff9350103fbb80 x1855008913276672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076350.449900:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913276672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076350.449906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076350.449908:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076350.449927:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076350.449931:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076350.449943:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076350.449949:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076350/real 1769076350] req@ffff9350103fb480 x1855008913276800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076350.449958:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913276800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076350.449964:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076350.449966:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076350.449968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076350.449971:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076350.449981:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076350.449987:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076350/real 1769076350] req@ffff9350103faa00 x1855008913276928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076350.449996:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913276928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076350.450001:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076350.450003:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076350.450005:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913276928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076350.641174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076351.153144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076351.153183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076351.153212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076351.153241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076351.473228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076351.665162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076352.177195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076352.177203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076352.177285:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076352.177289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076352.497276:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076352.689203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076353.201153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076353.201235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076353.201246:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076353.201273:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076353.521295:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076353.713180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076354.225176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076354.225238:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076354.225303:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076354.225347:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076354.545256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076354.737146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076355.249264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076355.249291:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076355.249337:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076355.249341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076355.569224:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076355.569238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076355.569245:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076355.569256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076355.569259:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2061 00000100:00080000:0.0:1769076355.569298:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076355.569314:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076355.569369:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913277056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076355.569416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076355.569421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076355.569424:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076355.569430:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2061 00000100:00080000:0.0:1769076355.569434:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076355.569439:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076355.569453:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076355.569473:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913277184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076355.569489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076355.569494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076355.569498:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076355.569503:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2061 00000100:00080000:0.0:1769076355.569507:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076355.569515:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076355.569524:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913277056:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769076355.569526:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076355.569566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913277312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076355.569596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076355.569605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076355.569610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076355.569619:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2061 00000100:00080000:0.0:1769076355.569628:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076355.569637:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076355.569650:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769076355.569661:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076355.569681:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913277440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076355.569699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076355.569704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076355.569708:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076355.569717:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2061 00000100:00080000:0.0:1769076355.569726:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076355.569737:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076355/real 1769076355] req@ffff9350103f9f80 x1855008913277056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076355.569745:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076355.569769:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076355.569792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913277056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076355.569795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913277568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076355.569884:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076355.569885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076355.569909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076355.569915:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076355.569917:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076355.569958:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2061 00000100:00080000:0.0:1769076355.569966:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076355.569971:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913277056:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076355.569981:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076355.569996:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076355.570001:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913277184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076355.570024:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913277696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076355.570040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076355.570096:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076355.570114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076355.570121:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076355/real 1769076355] req@ffff9350103fb800 x1855008913277184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076355.570124:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076355.570158:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2061 00000100:00100000:2.0:1769076355.570170:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913277184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076355.570175:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076355.570200:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076355.570203:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769076355.570216:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076355.570216:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076355.570238:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913277184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076355.570257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913277824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076355.570300:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913277312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076355.570330:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076355.570344:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076355/real 1769076355] req@ffff9350103fad80 x1855008913277312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076355.570360:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913277312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076355.570370:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076355.570374:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076355.570379:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913277312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076355.570386:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913277440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076355.570404:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076355.570414:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076355/real 1769076355] req@ffff9350103f9500 x1855008913277440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076355.570429:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913277440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076355.570448:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076355.570452:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076355.570456:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913277440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076355.570463:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913277568:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076355.570481:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076355.570492:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076355/real 1769076355] req@ffff9350103f9c00 x1855008913277568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076355.570505:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913277568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076355.570516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076355.570519:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076355.570523:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913277568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076355.570530:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913277696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076355.570549:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076355.570559:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076355/real 1769076355] req@ffff9350103f9180 x1855008913277696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076355.570574:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913277696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076355.570584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076355.570587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076355.570592:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913277696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076355.570598:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913277824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076355.570616:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076355.570626:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076355/real 1769076355] req@ffff9350103faa00 x1855008913277824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076355.570639:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913277824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076355.570648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076355.570651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076355.570655:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913277824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076355.761130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076356.273181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076356.273197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076356.273227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076356.273229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076356.593206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076356.785137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076357.297183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076357.297211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076357.297230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076357.297256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076357.617184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076357.809153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076358.321234:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076358.321236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076358.321278:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076358.321384:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076358.641236:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076358.833162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076359.345172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076359.345200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076359.345206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076359.345395:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076359.665180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076359.857159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076360.369138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076360.369182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076360.369189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076360.369318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076360.689189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076360.689271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076360.689283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076360.689289:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076360.689302:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2066 00000100:00080000:2.0:1769076360.689317:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076360.689325:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076360.689378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913277952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076360.689422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076360.689427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076360.689430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076360.689435:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2066 00000100:00080000:2.0:1769076360.689438:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076360.689444:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076360.689456:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076360.689473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913278080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076360.689486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076360.689491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076360.689493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076360.689497:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2066 00000100:00080000:2.0:1769076360.689503:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076360.689512:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076360.689514:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913277952:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076360.689519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076360.689537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913278208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076360.689551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076360.689555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076360.689557:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076360.689561:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2066 00000100:00080000:2.0:1769076360.689564:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076360.689567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076360.689573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076360.689583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913278336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076360.689594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076360.689600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076360.689603:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076360.689608:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2066 00000100:00080000:2.0:1769076360.689612:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076360.689617:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076360.689622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076360.689636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913278464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076360.689651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076360.689658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076360.689663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076360.689665:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076360.689670:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2066 00000100:00080000:2.0:1769076360.689677:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076360.689684:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076360.689690:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076360.689704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913278592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076360.689716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076360.689721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076360.689723:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076360.689728:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2066 00000100:00080000:2.0:1769076360.689732:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076360.689741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076360/real 1769076360] req@ffff935002a61c00 x1855008913277952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076360.689742:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076360.689754:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076360.689771:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913278720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076360.689803:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913277952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076360.689825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076360.689833:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076360.689861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913277952:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076360.689875:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278080:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076360.689918:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076360.689939:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076360/real 1769076360] req@ffff935002a61f80 x1855008913278080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076360.689959:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913278080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076360.689973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076360.689979:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076360.689984:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076360.690021:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076360.690090:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076360.690116:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076360/real 1769076360] req@ffff935002a63100 x1855008913278208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076360.690137:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913278208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076360.690150:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076360.690156:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076360.690161:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076360.690171:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076360.690201:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076360.690219:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076360/real 1769076360] req@ffff935002a61180 x1855008913278336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076360.690238:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913278336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076360.690251:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076360.690257:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076360.690262:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076360.690272:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913278464:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076360.690300:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076360.690317:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076360/real 1769076360] req@ffff935002a61500 x1855008913278464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076360.690336:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913278464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076360.690349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076360.690355:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076360.690360:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913278464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076360.690370:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913278592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076360.690397:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076360.690413:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076360/real 1769076360] req@ffff935002a62300 x1855008913278592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076360.690430:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913278592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076360.690443:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076360.690448:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076360.690454:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913278592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076360.690463:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913278720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076360.690487:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076360.690502:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076360/real 1769076360] req@ffff935002a60a80 x1855008913278720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076360.690529:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913278720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076360.690542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076360.690547:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076360.690552:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913278720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076360.881145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076361.393166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076361.393166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076361.393172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076361.393239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076361.713140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076361.905140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076362.417212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076362.417213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076362.417213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076362.417294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076362.737139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076362.929193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076363.441172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076363.441194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076363.441211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076363.441233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076363.761192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076363.953182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076364.465177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076364.465186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076364.465196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076364.465321:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076364.785207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076364.977180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076365.489214:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076365.489266:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076365.489273:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076365.489358:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076365.809118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076365.809128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076365.809135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076365.809140:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076365.809149:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2071 00000100:00080000:2.0:1769076365.809161:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076365.809169:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076365.809209:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913278848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076365.809235:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076365.809238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076365.809239:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076365.809243:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2071 00000100:00080000:2.0:1769076365.809245:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076365.809247:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076365.809255:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076365.809264:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913278976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076365.809276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076365.809280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076365.809283:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076365.809285:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2071 00000100:00080000:2.0:1769076365.809287:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076365.809290:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076365.809294:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076365.809306:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913279104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076365.809317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076365.809321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076365.809323:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076365.809328:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2071 00000100:00080000:2.0:1769076365.809332:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076365.809337:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076365.809352:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913278848:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076365.809357:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076365.809384:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913279232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076365.809399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076365.809403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076365.809405:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076365.809409:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2071 00000100:00080000:2.0:1769076365.809415:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076365.809420:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076365.809425:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076365.809436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913279360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076365.809445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076365.809449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076365.809450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076365.809452:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2071 00000100:00080000:2.0:1769076365.809455:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076365.809457:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076365.809462:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076365.809465:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076365.809480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913279488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076365.809493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076365.809497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076365.809498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076365.809504:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2071 00000100:00080000:2.0:1769076365.809507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076365.809509:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076365.809513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076365.809526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913279616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076365.809542:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076365/real 1769076365] req@ffff935002a62680 x1855008913278848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076365.809571:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913278848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076365.809587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076365.809593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076365.809599:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913278848:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076365.809609:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278976:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076365.809637:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076365.809651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076365/real 1769076365] req@ffff935002a61880 x1855008913278976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076365.809668:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913278976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076365.809679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076365.809683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076365.809687:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913278976:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076365.809694:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913279104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076365.809714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076365.809725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076365/real 1769076365] req@ffff935002a62a00 x1855008913279104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076365.809739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913279104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076365.809749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076365.809782:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076365.809787:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913279104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076365.809826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913279232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076365.809854:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076365.809869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076365/real 1769076365] req@ffff935002a60000 x1855008913279232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076365.809884:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913279232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076365.809902:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076365.809906:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076365.809910:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913279232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076365.809917:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913279360:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076365.809935:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076365.809947:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076365/real 1769076365] req@ffff935002a60a80 x1855008913279360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076365.809960:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913279360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076365.809971:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076365.809974:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076365.809978:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913279360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076365.809984:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913279488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076365.810001:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076365.810012:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076365/real 1769076365] req@ffff935002a62300 x1855008913279488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076365.810026:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913279488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076365.810036:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076365.810039:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076365.810043:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913279488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076365.810112:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913279616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076365.810157:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076365.810182:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076365/real 1769076365] req@ffff935002a61500 x1855008913279616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076365.810207:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913279616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076365.810218:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076365.810222:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076365.810226:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913279616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076366.001150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076366.513142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076366.513157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076366.513251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076366.513262:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076366.833144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076367.025168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076367.537205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076367.537218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076367.537248:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076367.537329:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076367.857185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076368.049146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076368.561189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076368.561218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076368.561247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076368.561373:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076368.881207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076369.073151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076369.585171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076369.585222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076369.585248:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076369.585338:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076369.905198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076370.097142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076370.609188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076370.609198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076370.609280:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076370.609401:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076370.929175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076370.929193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076370.929200:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076370.929213:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2076 00000100:00100000:1.0:1769076370.929231:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076370.929232:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076370.929248:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076370.929303:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913279744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076370.929344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076370.929350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076370.929353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076370.929359:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2076 00000100:00080000:2.0:1769076370.929363:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076370.929367:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076370.929380:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076370.929395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913279872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076370.929409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076370.929414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076370.929416:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076370.929422:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2076 00000100:00080000:2.0:1769076370.929428:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076370.929433:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076370.929439:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076370.929452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913280000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076370.929466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076370.929474:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913279744:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076370.929474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076370.929478:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076370.929491:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2076 00000100:00080000:2.0:1769076370.929511:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076370.929517:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076370.929524:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076370.929540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913280128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076370.929553:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076370.929558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076370.929562:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076370.929566:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2076 00000100:00080000:2.0:1769076370.929570:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076370.929575:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076370.929583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076370.929605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913280256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076370.929606:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076370.929634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076370.929639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076370.929643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076370.929658:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2076 00000100:00080000:2.0:1769076370.929663:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076370.929691:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076370.929698:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076370.929716:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff935002a61180 x1855008913279744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076370.929720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913280384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076370.929763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076370.929774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076370.929776:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076370.929783:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2076 00000100:00080000:2.0:1769076370.929789:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076370.929799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913279744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076370.929799:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076370.929812:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076370.929831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076370.929834:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913280512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076370.929846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076370.929863:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913279744:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076370.929876:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913279872:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076370.929914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076370.929933:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff935002a63100 x1855008913279872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.929949:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913279872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.929961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.929965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076370.929970:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913279872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076370.930002:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076370.930023:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076370.930036:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff935002a61f80 x1855008913280000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.930115:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913280000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.930128:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.930133:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076370.930137:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076370.930146:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076370.930171:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076370.930185:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff935002a61c00 x1855008913280128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.930201:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913280128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.930212:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.930216:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076370.930220:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076370.930226:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913280256:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076370.930244:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076370.930256:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff935002a61500 x1855008913280256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.930271:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913280256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.930281:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.930285:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076370.930288:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913280256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076370.930295:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913280384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076370.930313:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076370.930324:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff935002a62300 x1855008913280384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.930338:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913280384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.930362:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.930368:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076370.930375:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913280384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076370.930386:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913280512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076370.930422:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076370.930446:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff935002a60a80 x1855008913280512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.930471:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913280512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.930488:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.930493:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076370.930497:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913280512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076371.121167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076371.633193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076371.633197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076371.633215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076371.633270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076371.953243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076372.145177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076372.657152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076372.657155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076372.657179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076372.657250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076372.977242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076373.169193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076373.681200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076373.681229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076373.681229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076373.681280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076374.001168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076374.193139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076374.705165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076374.705196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076374.705212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076374.705260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076375.025172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076375.217152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076375.729172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076375.729190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076375.729208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076375.729283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076376.049191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076376.049229:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076376.049250:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076376.049260:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076376.049279:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2081 00000100:00080000:2.0:1769076376.049295:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076376.049305:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076376.049367:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913280640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076376.049421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076376.049430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076376.049434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076376.049443:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2081 00000100:00080000:2.0:1769076376.049452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076376.049463:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076376.049483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076376.049498:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913280640:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076376.049511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913280768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076376.049530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076376.049535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076376.049538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076376.049544:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2081 00000100:00080000:2.0:1769076376.049550:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076376.049555:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076376.049563:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076376.049578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913280896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076376.049626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076376.049629:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076376.049638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076376.049642:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076376.049650:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2081 00000100:00080000:2.0:1769076376.049658:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076376.049664:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076376.049673:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076376.049691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913281024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076376.049705:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076376.049711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076376.049714:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076376.049724:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076376/real 1769076376] req@ffff935002a60000 x1855008913280640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076376.049724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2081 00000100:00080000:2.0:1769076376.049733:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076376.049742:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076376.049753:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076376.049777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913281152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076376.049791:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913280640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076376.049795:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076376.049811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076376.049817:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076376.049829:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2081 00000100:00080000:0.0:1769076376.049835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076376.049837:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076376.049849:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076376.049854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769076376.049858:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076376.049866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913280640:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076376.049879:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913281280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076376.049902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076376.049906:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076376.049909:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076376.049915:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2081 00000100:00080000:2.0:1769076376.049920:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076376.049924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280768:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076376.049932:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076376.049942:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076376.049961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913281408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076376.049995:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076376.050028:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076376/real 1769076376] req@ffff935002a62a00 x1855008913280768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076376.050117:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913280768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076376.050130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076376.050135:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076376.050139:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076376.050149:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076376.050188:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076376.050204:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076376/real 1769076376] req@ffff935002a61880 x1855008913280896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076376.050218:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913280896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076376.050229:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076376.050233:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076376.050237:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913280896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076376.050244:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076376.050263:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076376.050274:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076376/real 1769076376] req@ffff935002a62680 x1855008913281024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076376.050288:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913281024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076376.050297:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076376.050301:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076376.050305:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076376.050311:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913281152:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076376.050329:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076376.050340:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076376/real 1769076376] req@ffff935002a60a80 x1855008913281152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076376.050354:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913281152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076376.050364:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076376.050367:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076376.050370:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913281152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076376.050377:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913281280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076376.050395:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076376.050406:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076376/real 1769076376] req@ffff935002a62300 x1855008913281280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076376.050421:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913281280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076376.050430:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076376.050433:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076376.050437:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913281280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076376.050463:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913281408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076376.050481:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076376.050492:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076376/real 1769076376] req@ffff935002a61500 x1855008913281408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076376.050505:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913281408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076376.050515:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076376.050518:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076376.050522:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913281408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076376.241171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076376.753141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076376.753205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076376.753226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076376.753261:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076377.073259:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076377.265179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076377.777244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076377.777247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076377.777272:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076377.777291:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076378.097273:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076378.289161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076378.801163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076378.801226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076378.801267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076378.801288:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076379.121209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076379.313194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076379.825220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076379.825241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076379.825287:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076379.825306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076380.145187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076380.337158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076380.849159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076380.849225:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076380.849293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076380.849298:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769076381.169211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076381.169228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076381.169235:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076381.169245:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076381.169252:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2086 00000100:00080000:3.0:1769076381.169274:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076381.169289:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076381.169358:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913281536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076381.169411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076381.169417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076381.169420:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076381.169426:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2086 00000100:00080000:3.0:1769076381.169432:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076381.169438:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076381.169452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076381.169470:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913281664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076381.169485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076381.169490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076381.169520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076381.169528:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2086 00000100:00080000:3.0:1769076381.169534:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076381.169539:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076381.169548:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076381.169561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913281536:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076381.169568:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913281792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076381.169587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076381.169592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076381.169595:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076381.169601:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2086 00000100:00080000:3.0:1769076381.169605:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076381.169610:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076381.169616:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076381.169638:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913281920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076381.169656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076381.169664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076381.169671:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076381.169672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076381.169689:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2086 00000100:00080000:3.0:1769076381.169700:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076381.169710:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076381.169722:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076381.169747:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076381/real 1769076381] req@ffff935002a7e300 x1855008913281536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076381.169756:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913282048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076381.169798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076381.169806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076381.169807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913281536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076381.169811:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076381.169826:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2086 00000100:00080000:3.0:1769076381.169837:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076381.169844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076381.169848:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076381.169857:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769076381.169859:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076381.169866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913281536:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076381.169886:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913282176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076381.169887:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281664:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076381.169910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076381.169915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076381.169918:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076381.169925:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2086 00000100:00080000:3.0:1769076381.169931:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076381.169932:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076381.169940:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076381.169949:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076381.169960:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076381/real 1769076381] req@ffff935002a7f100 x1855008913281664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076381.169970:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913282304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076381.169996:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913281664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076381.170010:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076381.170016:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076381.170022:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076381.170107:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076381.170134:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076381.170149:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076381/real 1769076381] req@ffff935002a7c380 x1855008913281792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076381.170165:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913281792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076381.170183:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076381.170187:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076381.170191:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076381.170199:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076381.170217:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076381.170229:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076381/real 1769076381] req@ffff935002a7ca80 x1855008913281920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076381.170243:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913281920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076381.170253:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076381.170257:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076381.170260:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913281920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076381.170267:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282048:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076381.170285:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076381.170296:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076381/real 1769076381] req@ffff935002a7fb80 x1855008913282048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076381.170310:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913282048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076381.170320:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076381.170323:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076381.170328:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076381.170335:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076381.170352:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076381.170363:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076381/real 1769076381] req@ffff935002a7c700 x1855008913282176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076381.170376:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913282176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076381.170386:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076381.170389:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076381.170393:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076381.170399:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076381.170415:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076381.170425:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076381/real 1769076381] req@ffff935002a7d180 x1855008913282304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076381.170438:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913282304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076381.170447:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076381.170451:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076381.170455:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076381.361139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076381.873225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076381.873236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076381.873286:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076381.873380:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076382.193147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076382.385125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076382.897158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076382.897166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076382.897167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076382.897210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076383.217149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076383.409191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076383.921190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076383.921191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076383.921209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076383.921285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076384.241164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076384.433214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076384.945216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076384.945219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076384.945233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076384.945305:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076385.265134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076385.457174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076385.969176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076385.969181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076385.969183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076385.969333:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076386.289185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076386.289230:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076386.289242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076386.289250:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076386.289262:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2091 00000100:00080000:0.0:1769076386.289279:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076386.289289:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076386.289344:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913282432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076386.289388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076386.289393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076386.289396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076386.289402:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2091 00000100:00080000:0.0:1769076386.289406:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076386.289437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076386.289450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076386.289472:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913282560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076386.289487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076386.289493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076386.289496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076386.289504:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2091 00000100:00080000:0.0:1769076386.289510:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076386.289516:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076386.289523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076386.289535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913282688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076386.289552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076386.289560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076386.289568:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076386.289571:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913282432:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076386.289581:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2091 00000100:00080000:0.0:1769076386.289594:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076386.289602:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076386.289607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076386.289630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913282816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076386.289643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076386.289648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076386.289651:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076386.289658:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2091 00000100:00080000:0.0:1769076386.289665:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076386.289669:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076386.289675:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076386.289688:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913282944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076386.289702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076386.289706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076386.289709:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076386.289713:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2091 00000100:00080000:0.0:1769076386.289717:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076386.289723:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076386.289728:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076386.289746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913283072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076386.289759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076386.289765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076386.289770:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076386.289780:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2091 00000100:00100000:2.0:1769076386.289792:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913282560:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076386.289796:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076386.289816:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076386.289830:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076386.289863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913283200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076386.289904:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913282688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076386.289968:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913282816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076386.290025:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076386.290202:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076386.290235:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076386.481159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076386.993210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076386.993251:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076386.993258:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076386.993442:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076387.313235:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076387.505187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076388.017185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076388.017193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076388.017216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076388.017306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076388.337184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076388.529209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076389.041189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076389.041210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076389.041287:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076389.041391:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076389.361180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769076389.361474:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769076389.361534:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076389.361561:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076389.361618:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076389.361628:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076389.361638:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076389.361647:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076389.361667:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076389.361682:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769076389.361730:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076386/real 1769076389] req@ffff9350103fb100 x1855008913282688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076389.361781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913282688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076389.361798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076389.361809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076389.361818:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913282688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076389.361834:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076386/real 1769076389] req@ffff9350103f8380 x1855008913282816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076389.361859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913282816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076389.361877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076389.361883:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076389.361889:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913282816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076389.361904:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076386/real 1769076389] req@ffff9350103f8700 x1855008913282944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076389.361940:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913282944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076389.361971:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076389.361978:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076389.361985:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913282944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076389.362000:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076386/real 1769076389] req@ffff9350103faa00 x1855008913283072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076389.362027:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913283072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076389.362042:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076389.362101:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076389.362111:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076389.362125:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076386/real 1769076389] req@ffff9350103f9180 x1855008913283200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076389.362148:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913283200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076389.362174:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076389.362182:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076389.362188:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076389.362202:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076386/real 1769076389] req@ffff9350103fb480 x1855008913282432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076389.362228:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913282432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076389.362244:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076389.362249:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076389.362256:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913282432:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076389.362267:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076386/real 1769076389] req@ffff9350103fbb80 x1855008913282560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076389.362301:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913282560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076389.362319:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076389.362326:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076389.362332:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913282560:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076389.553164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076390.065204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076390.065213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076390.065269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076390.065301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076390.385194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076390.577153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076391.089179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076391.089225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076391.089240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076391.089377:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076391.409168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076391.409225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076391.409236:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076391.409243:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076391.409258:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2097 00000100:00080000:0.0:1769076391.409273:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076391.409283:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076391.409369:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913283328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076391.409412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076391.409417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076391.409420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076391.409425:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2097 00000100:00080000:0.0:1769076391.409430:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076391.409435:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076391.409449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076391.409466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913283456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076391.409480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076391.409485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076391.409487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076391.409492:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2097 00000100:00080000:0.0:1769076391.409496:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076391.409500:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076391.409507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076391.409521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913283584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076391.409534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076391.409539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076391.409541:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076391.409545:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2097 00000100:00080000:0.0:1769076391.409550:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076391.409561:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076391.409562:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913283328:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769076391.409567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076391.409586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913283712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076391.409619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076391.409628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076391.409632:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076391.409641:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2097 00000100:00080000:0.0:1769076391.409649:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076391.409657:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076391.409668:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076391.409691:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076391.409696:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913283840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076391.409719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076391.409727:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076391.409731:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076391.409740:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2097 00000100:00080000:0.0:1769076391.409747:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076391.409756:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076391.409767:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076391.409789:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076391/real 1769076391] req@ffff9350103f9c00 x1855008913283328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076391.409797:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913283968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076391.409861:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913283328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076391.409888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076391.409893:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076391.409900:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076391.409908:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076391.409910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076391.409925:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2097 00000100:00100000:2.0:1769076391.409932:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913283328:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076391.409938:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076391.409956:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076391.409968:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076391.409968:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913283456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076391.410000:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913284096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076391.410040:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076391.410161:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076391/real 1769076391] req@ffff9350103f9500 x1855008913283456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076391.410192:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913283456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076391.410211:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076391.410217:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076391.410224:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913283456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076391.410236:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913283584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076391.410279:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076391.410305:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076391/real 1769076391] req@ffff9350103fad80 x1855008913283584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076391.410366:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913283584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076391.410384:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076391.410391:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076391.410398:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913283584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076391.410450:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913283712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076391.410493:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076391.410532:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076391/real 1769076391] req@ffff9350103fb800 x1855008913283712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076391.410559:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913283712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076391.410577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076391.410583:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076391.410590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913283712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076391.410603:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283840:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076391.410643:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076391.410667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076391/real 1769076391] req@ffff9350103f9f80 x1855008913283840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076391.410693:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913283840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076391.410710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076391.410716:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076391.410723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283840:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076391.410737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076391.410791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076391.410815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076391/real 1769076391] req@ffff9350103fbb80 x1855008913283968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076391.410837:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913283968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076391.410866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076391.410873:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076391.410879:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913283968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076391.410892:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076391.410930:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076391.410954:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076391/real 1769076391] req@ffff9350103fb480 x1855008913284096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076391.410974:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913284096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076391.411001:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076391.411007:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076391.411013:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076391.601181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076392.113213:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076392.113224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076392.113274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076392.113338:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076392.433176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076392.625180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076393.137217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076393.137217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076393.137260:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076393.137344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076393.457154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076393.649190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076394.161187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076394.161226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076394.161309:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076394.161378:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076394.481181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076394.673172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076395.185157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076395.185173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076395.185181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076395.185406:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076395.505243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076395.697153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076396.209177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076396.209218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076396.209265:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076396.209363:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076396.529134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076396.529341:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076396.529372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076396.529379:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076396.529393:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2102 00000100:00080000:2.0:1769076396.529408:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076396.529417:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076396.529476:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913284224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076396.529526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076396.529531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076396.529533:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076396.529539:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2102 00000100:00080000:2.0:1769076396.529545:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076396.529550:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076396.529565:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076396.529589:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913284224:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076396.529591:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913284352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076396.529633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076396.529655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076396.529659:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076396.529667:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2102 00000100:00080000:2.0:1769076396.529673:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076396.529690:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076396.529698:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076396.529712:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913284480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076396.529721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076396.529725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076396.529727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076396.529731:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2102 00000100:00080000:2.0:1769076396.529735:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076396.529740:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076396.529756:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076396.529760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076396.529772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913284608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076396.529782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076396.529786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076396.529788:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076396.529793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2102 00000100:00080000:2.0:1769076396.529796:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076396.529799:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076396.529804:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076396.529816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076396/real 1769076396] req@ffff935002a61c00 x1855008913284224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076396.529817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913284736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076396.529866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076396.529879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076396.529884:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076396.529897:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2102 00000100:00080000:2.0:1769076396.529909:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076396.529919:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076396.529924:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076396.529945:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913284224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076396.529958:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913284864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076396.529964:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076396.529986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076396.529994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076396.530003:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913284224:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076396.530007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076396.530019:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076396.530027:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2102 00000100:00080000:2.0:1769076396.530033:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076396.530048:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076396.530055:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076396.530066:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913284352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076396.530074:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913284992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076396.530172:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076396.530204:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076396/real 1769076396] req@ffff935002a61f80 x1855008913284352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076396.530218:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913284352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076396.530227:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076396.530230:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076396.530234:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913284352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076396.530262:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913284480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076396.530295:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076396.530305:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076396/real 1769076396] req@ffff935002a63100 x1855008913284480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076396.530316:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913284480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076396.530329:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076396.530333:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076396.530335:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913284480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076396.530341:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913284608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076396.530367:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076396.530380:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076396/real 1769076396] req@ffff935002a61180 x1855008913284608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076396.530396:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913284608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076396.530408:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076396.530413:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076396.530418:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913284608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076396.530428:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284736:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076396.530454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076396.530466:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076396/real 1769076396] req@ffff935002a61500 x1855008913284736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076396.530477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913284736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076396.530484:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076396.530487:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076396.530491:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076396.530497:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076396.530513:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076396.530521:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076396/real 1769076396] req@ffff935002a62300 x1855008913284864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076396.530531:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913284864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076396.530538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076396.530540:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076396.530543:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076396.530565:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076396.530579:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076396.530588:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076396/real 1769076396] req@ffff935002a60a80 x1855008913284992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076396.530598:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913284992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076396.530605:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076396.530607:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076396.530610:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913284992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076396.721124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076397.233176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076397.233189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076397.233223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076397.233350:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076397.553202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076397.745149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076398.257231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076398.257250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076398.257251:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076398.257317:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076398.577191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076398.769153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076399.281267:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076399.281269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076399.281282:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076399.281374:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076399.601247:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076399.793160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076400.305151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076400.305219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076400.305225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076400.305374:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076400.625161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076400.817189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076401.329212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076401.329217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076401.329227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076401.329291:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076401.649188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076401.649243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076401.649255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076401.649261:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076401.649273:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2107 00000100:00080000:3.0:1769076401.649288:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076401.649298:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076401.649349:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913285120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076401.649395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076401.649400:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076401.649402:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076401.649407:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2107 00000100:00080000:3.0:1769076401.649411:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076401.649415:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076401.649445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076401.649464:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913285248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076401.649480:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076401.649486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076401.649509:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913285120:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076401.649509:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076401.649518:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2107 00000100:00080000:3.0:1769076401.649526:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076401.649531:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076401.649537:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076401.649554:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913285376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076401.649567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076401.649571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076401.649574:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076401.649578:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2107 00000100:00080000:3.0:1769076401.649584:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076401.649588:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076401.649594:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076401.649610:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913285504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076401.649623:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076401.649636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076401.649644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076401.649648:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076401.649658:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2107 00000100:00080000:3.0:1769076401.649669:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076401.649677:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076401.649687:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076401.649708:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076401/real 1769076401] req@ffff935002a7dc00 x1855008913285120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076401.649718:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913285632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076401.649764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076401.649771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076401.649780:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913285120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076401.649780:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076401.649796:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2107 00000100:00080000:3.0:1769076401.649809:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076401.649815:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076401.649823:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076401.649832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076401.649843:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769076401.649849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913285760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076401.649856:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913285120:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076401.649887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076401.649890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076401.649893:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076401.649899:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2107 00000100:00100000:0.0:1769076401.649906:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913285248:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076401.649908:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076401.649921:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076401.649930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076401.649947:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913285888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076401.649972:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076401.649999:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076401/real 1769076401] req@ffff935002a7f800 x1855008913285248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076401.650016:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913285248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076401.650028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076401.650032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076401.650039:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913285248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076401.650101:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913285376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076401.650176:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076401.650197:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076401/real 1769076401] req@ffff935002a7c000 x1855008913285376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076401.650214:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913285376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076401.650234:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076401.650239:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076401.650243:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913285376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076401.650251:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913285504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076401.650277:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076401.650289:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076401/real 1769076401] req@ffff935002a7df80 x1855008913285504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076401.650303:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913285504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076401.650313:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076401.650316:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076401.650319:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913285504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076401.650326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913285632:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076401.650344:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076401.650355:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076401/real 1769076401] req@ffff935002a7d180 x1855008913285632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076401.650377:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913285632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076401.650387:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076401.650390:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076401.650394:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913285632:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076401.650432:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913285760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076401.650463:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076401.650476:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076401/real 1769076401] req@ffff935002a7c700 x1855008913285760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076401.650492:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913285760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076401.650502:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076401.650506:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076401.650510:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913285760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076401.650557:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913285888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076401.650576:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076401.650588:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076401/real 1769076401] req@ffff935002a7fb80 x1855008913285888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076401.650601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913285888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076401.650618:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076401.650622:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076401.650626:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913285888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076401.841168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076402.353224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076402.353239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076402.353239:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076402.353316:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076402.673169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076402.865211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076403.377141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076403.377184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076403.377228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076403.377250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076403.697155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076403.889108:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076404.401120:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076404.401135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076404.401139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076404.401193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076404.721153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076404.913215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076405.425223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076405.425232:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076405.425312:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076405.425426:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076405.745171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076405.937176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076406.449209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076406.449210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076406.449223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076406.449284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076406.769182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076406.769271:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076406.769282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076406.769288:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076406.769300:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2112 00000100:00080000:2.0:1769076406.769316:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076406.769326:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076406.769380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913286016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076406.769421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076406.769426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076406.769429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076406.769435:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2112 00000100:00080000:2.0:1769076406.769440:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076406.769445:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076406.769458:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076406.769475:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913286144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076406.769491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076406.769496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076406.769502:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076406.769505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913286016:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076406.769513:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2112 00000100:00080000:2.0:1769076406.769525:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076406.769531:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076406.769538:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076406.769557:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913286272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076406.769569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076406.769574:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076406.769577:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076406.769582:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2112 00000100:00080000:2.0:1769076406.769587:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076406.769591:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076406.769597:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076406.769611:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913286400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076406.769626:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076406.769644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076406.769650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076406.769654:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076406.769661:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2112 00000100:00080000:2.0:1769076406.769667:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076406.769671:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076406.769677:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076406.769701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076406/real 1769076406] req@ffff935002a62680 x1855008913286016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076406.769703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913286528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076406.769761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076406.769768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076406.769771:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076406.769777:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2112 00000100:00100000:0.0:1769076406.769785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913286016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076406.769786:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076406.769800:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076406.769814:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076406.769824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076406.769836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913286656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076406.769837:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076406.769853:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913286016:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076406.769872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076406.769878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076406.769880:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076406.769886:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2112 00000100:00080000:2.0:1769076406.769893:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076406.769898:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076406.769906:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076406.769908:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913286144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076406.769924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913286784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076406.769973:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076406.770004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076406/real 1769076406] req@ffff935002a61880 x1855008913286144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076406.770031:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913286144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076406.770045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076406.770125:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076406.770132:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913286144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076406.770142:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913286272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076406.770178:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076406.770203:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076406/real 1769076406] req@ffff935002a62a00 x1855008913286272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076406.770227:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913286272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076406.770245:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076406.770252:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076406.770259:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913286272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076406.770272:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913286400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076406.770312:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076406.770330:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076406/real 1769076406] req@ffff935002a60000 x1855008913286400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076406.770346:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913286400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076406.770357:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076406.770362:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076406.770370:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913286400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076406.770382:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913286528:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076406.770420:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076406.770443:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076406/real 1769076406] req@ffff935002a60a80 x1855008913286528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076406.770469:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913286528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076406.770487:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076406.770492:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076406.770497:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913286528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076406.770505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913286656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076406.770535:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076406.770550:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076406/real 1769076406] req@ffff935002a62300 x1855008913286656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076406.770565:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913286656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076406.770576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076406.770580:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076406.770584:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913286656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076406.770610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913286784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076406.770629:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076406.770642:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076406/real 1769076406] req@ffff935002a61500 x1855008913286784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076406.770657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913286784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076406.770667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076406.770670:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076406.770674:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913286784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076406.961152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076407.473162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076407.473218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076407.473218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076407.473306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076407.793197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076407.985155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076408.497191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076408.497196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076408.497201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076408.497326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076408.817180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076409.009217:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076409.521200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076409.521201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076409.521277:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076409.521331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076409.841202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076410.033176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076410.545217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076410.545225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076410.545225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076410.545286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076410.865210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076411.057182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076411.569218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076411.569235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076411.569285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076411.569322:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769076411.889182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076411.889199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076411.889206:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076411.889207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076411.889229:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2117 00000100:00080000:1.0:1769076411.889246:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076411.889257:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076411.889316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913286912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076411.889359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076411.889365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076411.889368:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076411.889373:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2117 00000100:00080000:1.0:1769076411.889378:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076411.889382:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076411.889397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076411.889415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913287040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076411.889441:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913286912:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076411.889444:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076411.889457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076411.889461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076411.889468:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2117 00000100:00080000:1.0:1769076411.889473:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076411.889477:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076411.889484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076411.889503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913287168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076411.889516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076411.889521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076411.889523:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076411.889528:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2117 00000100:00080000:1.0:1769076411.889534:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076411.889538:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076411.889543:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076411.889551:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076411.889560:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913287296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076411.889576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076411.889581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076411.889584:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076411.889589:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2117 00000100:00080000:1.0:1769076411.889595:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076411.889600:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076411.889609:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076411.889620:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff9350100c8380 x1855008913286912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076411.889635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913287424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076411.889655:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913286912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076411.889660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076411.889668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076411.889672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076411.889689:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2117 00000100:00080000:0.0:1769076411.889701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076411.889703:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076411.889716:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076411.889723:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076411.889730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913286912:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769076411.889740:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076411.889773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076411.889780:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913287552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076411.889805:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076411.889810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076411.889814:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076411.889820:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2117 00000800:00000100:0.0:1769076411.889829:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076411.889834:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076411.889844:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076411.889854:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076411.889862:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff9350100ca300 x1855008913287040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076411.889878:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913287680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076411.889891:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913287040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076411.889958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076411.889967:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076411.889972:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076411.890022:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076411.890110:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076411.890126:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff9350100ca680 x1855008913287168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076411.890161:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913287168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076411.890178:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076411.890185:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076411.890191:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076411.890204:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076411.890244:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076411.890275:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff9350100c8e00 x1855008913287296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076411.890309:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913287296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076411.890320:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076411.890324:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076411.890329:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076411.890336:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913287424:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076411.890365:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076411.890379:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff9350100c9880 x1855008913287424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076411.890394:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913287424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076411.890404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076411.890407:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076411.890411:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913287424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076411.890435:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913287552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076411.890454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076411.890466:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff9350100c9500 x1855008913287552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076411.890480:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913287552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076411.890498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076411.890502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076411.890506:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913287552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076411.890512:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913287680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076411.890528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076411.890540:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff9350100caa00 x1855008913287680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076411.890553:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913287680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076411.890563:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076411.890567:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076411.890571:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913287680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076412.081127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076412.593229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076412.593244:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076412.593264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076412.593273:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076412.913270:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076413.105158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076413.617229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076413.617256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076413.617264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076413.617311:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076413.937241:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076414.129151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076414.641220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076414.641237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076414.641286:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076414.641387:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076414.961169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076415.153137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076415.665178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076415.665179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076415.665244:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076415.665270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076415.985152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076416.177170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076416.689182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076416.689188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076416.689199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076416.689218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076417.009184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076417.009189:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076417.009204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076417.009210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076417.009223:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2122 00000100:00080000:1.0:1769076417.009240:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076417.009252:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076417.009303:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913287808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076417.009348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076417.009353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076417.009356:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076417.009361:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2122 00000100:00080000:1.0:1769076417.009366:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076417.009370:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076417.009383:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076417.009400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913287936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076417.009413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076417.009417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076417.009420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076417.009426:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2122 00000100:00080000:1.0:1769076417.009430:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076417.009436:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076417.009443:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076417.009456:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913288064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076417.009469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076417.009475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076417.009480:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913287808:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076417.009480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076417.009488:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2122 00000100:00080000:1.0:1769076417.009492:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076417.009496:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076417.009501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076417.009534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913288192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076417.009546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076417.009551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076417.009554:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076417.009559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2122 00000100:00080000:1.0:1769076417.009564:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076417.009569:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076417.009577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076417.009590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913288320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076417.009603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076417.009607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076417.009609:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076417.009613:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2122 00000100:00080000:1.0:1769076417.009619:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076417.009628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076417.009628:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076417.009641:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076417.009656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913288448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076417.009669:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076417.009674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076417.009677:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076417.009681:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2122 00000100:00080000:1.0:1769076417.009686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076417.009690:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076417.009698:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076417.009716:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076417/real 1769076417] req@ffff9350100cb800 x1855008913287808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076417.009726:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913288576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076417.009764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913287808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076417.009782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076417.009789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076417.009797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913287808:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076417.009807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076417.009839:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076417.009855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076417/real 1769076417] req@ffff9350100caa00 x1855008913287936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076417.009872:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913287936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076417.009883:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076417.009887:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076417.009929:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913287936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076417.009966:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076417.009996:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076417.010011:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076417/real 1769076417] req@ffff9350100c9500 x1855008913288064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076417.010036:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913288064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076417.010080:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076417.010085:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076417.010090:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076417.010098:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076417.010122:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076417.010135:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076417/real 1769076417] req@ffff9350100c9880 x1855008913288192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076417.010150:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913288192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076417.010161:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076417.010165:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076417.010169:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076417.010175:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913288320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076417.010194:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076417.010205:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076417/real 1769076417] req@ffff9350100c8e00 x1855008913288320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076417.010219:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913288320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076417.010238:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076417.010242:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076417.010246:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913288320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076417.010252:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913288448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076417.010269:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076417.010279:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076417/real 1769076417] req@ffff9350100ca680 x1855008913288448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076417.010292:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913288448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076417.010301:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076417.010305:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076417.010308:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913288448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076417.010315:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913288576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076417.010331:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076417.010341:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076417/real 1769076417] req@ffff9350100ca300 x1855008913288576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076417.010354:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913288576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076417.010363:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076417.010366:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076417.010371:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913288576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076417.201139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076417.713190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076417.713223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076417.713225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076417.714115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076418.033178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076418.225193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076418.737183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076418.737183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076418.737199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076418.737257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076419.057181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076419.249176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076419.761189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076419.761209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076419.761222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076419.761273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076420.081152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076420.273159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076420.785273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076420.785290:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076420.785329:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076420.785343:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076421.105101:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076421.297106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076421.809178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076421.809188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076421.809229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076421.809360:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076422.129182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076422.129185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076422.129199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076422.129205:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076422.129218:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2127 00000100:00080000:1.0:1769076422.129234:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076422.129246:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076422.129299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913288704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076422.129339:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076422.129344:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076422.129347:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076422.129352:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2127 00000100:00080000:1.0:1769076422.129357:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076422.129362:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076422.129377:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076422.129400:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913288704:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076422.129400:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913288832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076422.129428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076422.129432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076422.129435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076422.129441:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2127 00000100:00080000:1.0:1769076422.129445:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076422.129449:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076422.129458:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076422.129474:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913288960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076422.129487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076422.129492:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076422.129494:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076422.129499:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2127 00000100:00080000:1.0:1769076422.129505:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076422.129509:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076422.129514:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076422.129527:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076422.129530:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913289088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076422.129546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076422.129552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076422.129555:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076422.129561:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2127 00000100:00080000:1.0:1769076422.129567:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076422.129571:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076422.129577:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076422.129595:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913289216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076422.129607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076422.129615:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076422/real 1769076422] req@ffff9350100c8380 x1855008913288704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076422.129615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076422.129622:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076422.129636:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2127 00000100:00080000:1.0:1769076422.129647:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076422.129656:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076422.129663:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076422.129686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913289344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076422.129688:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913288704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076422.129718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076422.129732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076422.129736:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076422.129742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076422.129742:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2127 00000100:00080000:1.0:1769076422.129750:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076422.129760:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076422.129761:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076422.129774:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076422.129776:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913288704:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076422.129792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913289472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076422.129851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076422.129894:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076422.129914:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076422/real 1769076422] req@ffff9350100c9f80 x1855008913288832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076422.129931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913288832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076422.129958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076422.129963:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076422.129968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076422.129977:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076422.130001:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076422.130015:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076422/real 1769076422] req@ffff9350100c9c00 x1855008913288960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076422.130030:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913288960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076422.130041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076422.130046:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076422.130088:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913288960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076422.130103:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076422.130146:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076422.130172:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076422/real 1769076422] req@ffff9350100c8a80 x1855008913289088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076422.130195:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913289088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076422.130211:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076422.130218:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076422.130226:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076422.130239:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913289216:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076422.130281:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076422.130305:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076422/real 1769076422] req@ffff9350100c8000 x1855008913289216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076422.130328:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913289216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076422.130339:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076422.130343:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076422.130348:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913289216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076422.130356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913289344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076422.130384:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076422.130397:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076422/real 1769076422] req@ffff9350100cb100 x1855008913289344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076422.130412:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913289344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076422.130422:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076422.130425:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076422.130429:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913289344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076422.130436:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913289472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076422.130454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076422.130468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076422/real 1769076422] req@ffff9350100c9180 x1855008913289472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076422.130491:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913289472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076422.130507:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076422.130513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076422.130520:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913289472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076422.321153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076422.833196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076422.833206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076422.833214:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076422.833320:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076423.153194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076423.345157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076423.857160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076423.857209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076423.857223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076423.857345:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076424.177187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076424.369162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076424.881185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076424.881193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076424.881210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076424.881250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076425.201162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076425.393162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076425.905208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076425.905227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076425.905271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076425.905284:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076426.225173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076426.417159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076426.929230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076426.929230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076426.929245:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076426.929276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076427.249218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076427.249235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076427.249251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076427.249258:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076427.249271:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2132 00000100:00080000:3.0:1769076427.249289:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076427.249317:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076427.249373:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913289600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076427.249423:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076427.249431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076427.249435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076427.249444:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2132 00000100:00080000:3.0:1769076427.249453:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076427.249460:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076427.249493:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076427.249501:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913289600:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076427.249522:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913289728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076427.249545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076427.249552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076427.249556:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076427.249566:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2132 00000100:00080000:3.0:1769076427.249592:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076427.249601:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076427.249615:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076427.249621:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076427.249655:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913289856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076427.249679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076427.249684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076427.249687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076427.249696:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076427/real 1769076427] req@ffff935002a7ca80 x1855008913289600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076427.249705:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2132 00000100:00080000:3.0:1769076427.249750:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076427.249759:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076427.249769:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076427.249773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913289600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076427.249792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076427.249795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913289984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076427.249810:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076427.249830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913289600:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076427.249833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076427.249851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076427.249855:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076427.249862:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2132 00000100:00080000:3.0:1769076427.249867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076427.249873:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076427.249879:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076427.249902:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076427.249903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913290112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076427.249941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076427.249946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076427.249950:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076427.249957:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2132 00000100:00080000:3.0:1769076427.249964:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076427.249988:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076427.249989:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076427.250008:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076427.250032:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076427/real 1769076427] req@ffff935002a7c380 x1855008913289728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076427.250115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913290240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076427.250129:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913289728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076427.250137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076427.250155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076427.250162:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076427.250180:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2132 00000100:00080000:3.0:1769076427.250200:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076427.250208:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076427.250213:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076427.250232:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076427.250234:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076427.250252:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076427.250263:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913290368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076427.250282:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076427.250349:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076427.250367:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076427/real 1769076427] req@ffff935002a7f100 x1855008913289856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076427.250402:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913289856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076427.250413:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076427.250417:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076427.250421:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076427.250429:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076427.250461:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076427.250473:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076427/real 1769076427] req@ffff935002a7e300 x1855008913289984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076427.250486:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913289984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076427.250495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076427.250498:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076427.250502:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913289984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076427.250551:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913290112:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076427.250574:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076427.250597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076427/real 1769076427] req@ffff935002a7ca80 x1855008913290112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076427.250612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913290112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076427.250622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076427.250626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076427.250629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913290112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076427.250637:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913290240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076427.250655:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076427.250665:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076427/real 1769076427] req@ffff935002a7fb80 x1855008913290240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076427.250679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913290240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076427.250689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076427.250693:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076427.250698:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913290240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076427.250704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913290368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076427.250748:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076427.250766:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076427/real 1769076427] req@ffff935002a7c700 x1855008913290368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076427.250799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913290368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076427.250808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076427.250812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076427.250815:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913290368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076427.441158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076427.953209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076427.953272:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076427.953324:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076427.953326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076428.273180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076428.465158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076428.977186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076428.977206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076428.977302:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076428.977499:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076429.297205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076429.489183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076430.001274:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076430.001274:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076430.001322:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076430.001385:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076430.321213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076430.513174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076431.025162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076431.025172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076431.025200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076431.025263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076431.345153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076431.537151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076432.049183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076432.049204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076432.049279:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076432.049339:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076432.369199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076432.369209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076432.369228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076432.369237:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076432.369256:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2138 00000100:00080000:2.0:1769076432.369276:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076432.369291:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076432.369355:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913290496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076432.369428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076432.369434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076432.369438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076432.369450:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2138 00000100:00080000:2.0:1769076432.369457:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076432.369465:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076432.369481:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076432.369486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913290496:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076432.369508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913290624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076432.369531:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076432.369536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076432.369539:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076432.369545:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2138 00000100:00080000:2.0:1769076432.369553:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076432.369559:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076432.369567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076432.369584:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913290752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076432.369598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076432.369612:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076432.369618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076432.369626:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076432.369662:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2138 00000100:00080000:2.0:1769076432.369675:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076432.369685:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076432.369698:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076432.369724:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076432/real 1769076432] req@ffff935002a61180 x1855008913290496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076432.369740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913290880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076432.369766:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913290496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076432.369771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076432.369798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076432.369812:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076432.369822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076432.369824:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2138 00000100:00080000:0.0:1769076432.369838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076432.369840:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076432.369851:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913290496:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076432.369853:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076432.369894:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076432.369913:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913290624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076432.369924:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913291008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076432.369948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076432.369953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076432.369963:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076432.369965:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076432.369987:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2138 00000100:00080000:2.0:1769076432.369999:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076432.370003:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076432/real 1769076432] req@ffff935002a63100 x1855008913290624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076432.370022:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076432.370035:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913290624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076432.370040:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076432.370102:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076432.370108:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076432.370115:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913290624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076432.370131:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913290752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076432.370161:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913291136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076432.370177:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076432.370182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076432.370189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076432.370194:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076432.370202:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076432/real 1769076432] req@ffff935002a61f80 x1855008913290752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076432.370211:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2138 00000100:00080000:2.0:1769076432.370238:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076432.370246:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913290752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076432.370264:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076432.370294:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769076432.370295:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076432.370305:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076432.370314:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913290752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076432.370326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913290880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076432.370344:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913291264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076432.370361:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076432.370379:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076432/real 1769076432] req@ffff935002a61c00 x1855008913290880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076432.370409:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913290880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076432.370421:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076432.370432:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076432.370438:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913290880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076432.370464:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291008:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076432.370484:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076432.370496:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076432/real 1769076432] req@ffff935002a61180 x1855008913291008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076432.370510:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913291008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076432.370520:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076432.370524:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076432.370528:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076432.370536:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076432.370555:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076432.370566:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076432/real 1769076432] req@ffff935002a61500 x1855008913291136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076432.370579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913291136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076432.370590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076432.370593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076432.370597:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076432.370604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076432.370621:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076432.370632:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076432/real 1769076432] req@ffff935002a62300 x1855008913291264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076432.370671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913291264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076432.370681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076432.370685:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076432.370689:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076432.561177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076433.073188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076433.073192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076433.073269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076433.073338:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076433.393198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076433.585149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076434.097183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076434.097192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076434.097236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076434.097302:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076434.417186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076434.609153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076435.121201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076435.121206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076435.121237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076435.121354:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076435.441172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076435.633155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076436.145184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076436.145215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076436.145231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076436.145287:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076436.465179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076436.657181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076437.169190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076437.169202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076437.169228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076437.169330:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076437.489175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076437.489208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076437.489220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076437.489227:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076437.489239:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2143 00000100:00080000:1.0:1769076437.489256:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076437.489269:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076437.489333:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913291392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076437.489387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076437.489394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076437.489397:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076437.489404:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2143 00000100:00080000:1.0:1769076437.489412:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076437.489419:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076437.489434:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076437.489441:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913291392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076437.489463:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913291520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076437.489482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076437.489487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076437.489489:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076437.489496:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2143 00000100:00080000:1.0:1769076437.489501:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076437.489506:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076437.489513:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076437.489528:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913291648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076437.489541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076437.489545:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076437.489574:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076437.489577:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076437.489584:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2143 00000100:00080000:1.0:1769076437.489592:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076437.489598:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076437.489607:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076437.489626:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913291776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076437.489640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076437.489645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076437.489647:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076437.489654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076437/real 1769076437] req@ffff9350100cad80 x1855008913291392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076437.489660:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2143 00000100:00080000:1.0:1769076437.489673:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076437.489686:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076437.489695:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076437.489709:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913291392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076437.489717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913291904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076437.489772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076437.489778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076437.489778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076437.489783:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076437.489796:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2143 00000100:00080000:1.0:1769076437.489800:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076437.489804:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076437.489810:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076437.489819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769076437.489826:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913292032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076437.489843:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913291392:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076437.489858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076437.489866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076437.489869:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076437.489875:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2143 00000100:00080000:1.0:1769076437.489882:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076437.489887:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076437.489895:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913291520:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769076437.489899:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076437.489922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913292160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076437.489952:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076437.489979:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076437/real 1769076437] req@ffff9350100c9180 x1855008913291520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076437.489996:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913291520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076437.490009:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076437.490013:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076437.490019:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913291520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076437.490028:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913291648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076437.490086:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076437.490103:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076437/real 1769076437] req@ffff9350100cb100 x1855008913291648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076437.490119:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913291648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076437.490129:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076437.490134:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076437.490138:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913291648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076437.490146:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913291776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076437.490170:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076437.490182:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076437/real 1769076437] req@ffff9350100c8000 x1855008913291776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076437.490197:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913291776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076437.490207:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076437.490211:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076437.490216:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913291776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076437.490222:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291904:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076437.490242:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076437.490254:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076437/real 1769076437] req@ffff9350100c8a80 x1855008913291904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076437.490268:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913291904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076437.490278:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076437.490282:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076437.490286:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913291904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076437.490292:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076437.490311:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076437.490328:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076437/real 1769076437] req@ffff9350100c9c00 x1855008913292032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076437.490351:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913292032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076437.490366:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076437.490372:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076437.490379:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076437.490419:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076437.490459:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076437.490478:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076437/real 1769076437] req@ffff9350100c9f80 x1855008913292160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076437.490494:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913292160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076437.490505:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076437.490509:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076437.490514:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076437.681184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076438.193137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076438.193138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076438.193145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076438.193194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076438.513257:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076438.705153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076439.217150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076439.217179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076439.217202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076439.217271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076439.537099:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076439.729203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076440.241167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076440.241179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076440.241235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076440.241344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076440.561160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076440.753187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076441.265186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076441.265216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076441.265237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076441.265281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076441.585176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076441.777187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076442.289165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076442.289168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076442.289172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076442.289240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076442.609125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076442.609178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076442.609184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076442.609188:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076442.609197:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2148 00000100:00080000:3.0:1769076442.609208:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076442.609214:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076442.609252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913292288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076442.609283:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076442.609286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076442.609288:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076442.609291:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2148 00000100:00080000:3.0:1769076442.609293:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076442.609296:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076442.609306:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076442.609317:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913292288:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076442.609317:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913292416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076442.609330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076442.609333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076442.609334:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076442.609337:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2148 00000100:00080000:3.0:1769076442.609340:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076442.609343:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076442.609346:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076442.609354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913292544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076442.609360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076442.609363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076442.609364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076442.609366:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2148 00000100:00080000:3.0:1769076442.609369:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076442.609372:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076442.609375:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076442.609383:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913292672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076442.609390:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076442.609390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076442.609395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076442.609397:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076442.609401:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2148 00000100:00080000:3.0:1769076442.609404:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076442.609407:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076442.609410:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076442.609417:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913292800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076442.609424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076442.609426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076442.609428:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076442.609431:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2148 00000100:00080000:3.0:1769076442.609434:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076442.609437:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076442/real 1769076442] req@ffff935002a7d180 x1855008913292288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076442.609438:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076442.609444:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076442.609457:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913292928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076442.609477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913292288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076442.609500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076442.609504:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076442.609508:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076442.609509:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076442.609512:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076442.609515:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913292288:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076442.609518:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2148 00000100:00080000:3.0:1769076442.609523:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076442.609526:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076442.609531:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076442.609543:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913292416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076442.609547:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913293056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076442.609568:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076442.609581:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076442/real 1769076442] req@ffff935002a7df80 x1855008913292416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076442.609590:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913292416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076442.609596:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076442.609598:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076442.609600:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913292416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076442.609604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913292544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076442.609616:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076442.609621:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076442/real 1769076442] req@ffff935002a7c000 x1855008913292544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076442.609629:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913292544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076442.609634:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076442.609635:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076442.609637:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913292544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076442.609640:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913292672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076442.609649:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076442.609655:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076442/real 1769076442] req@ffff935002a7f800 x1855008913292672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076442.609661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913292672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076442.609667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076442.609669:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076442.609671:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913292672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076442.609674:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292800:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076442.609683:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076442.609689:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076442/real 1769076442] req@ffff935002a7dc00 x1855008913292800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076442.609695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913292800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076442.609700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076442.609702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076442.609704:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076442.609707:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076442.609721:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076442.609731:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076442/real 1769076442] req@ffff935002a7c700 x1855008913292928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076442.609742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913292928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076442.609750:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076442.609754:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076442.609758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913292928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076442.609781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076442.609798:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076442.609807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076442/real 1769076442] req@ffff935002a7fb80 x1855008913293056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076442.609815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913293056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076442.609820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076442.609822:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076442.609824:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076442.801121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076443.313145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076443.313146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076443.313155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076443.313207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076443.633145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076443.825121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076444.337164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076444.337230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076444.337230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076444.337331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076444.657150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076444.849208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076445.361170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076445.361171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076445.361283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076445.361289:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076445.681154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076445.873179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076446.385194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076446.385208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076446.385292:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076446.385326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076446.705106:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076446.897216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076447.409110:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076447.409125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076447.409154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076447.409165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076447.729139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076447.729153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076447.729187:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076447.729200:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2153 00000100:00080000:0.0:1769076447.729216:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076447.729227:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076447.729276:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913293184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076447.729325:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076447.729330:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076447.729333:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076447.729340:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2153 00000100:00080000:0.0:1769076447.729347:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076447.729355:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076447.729372:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076447.729440:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913293312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076447.729455:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913293184:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076447.729466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076447.729474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076447.729477:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076447.729486:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2153 00000100:00080000:0.0:1769076447.729493:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076447.729498:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076447.729507:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076447.729528:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913293440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076447.729545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076447.729551:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076447.729557:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076447.729563:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076447.729573:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2153 00000100:00080000:0.0:1769076447.729582:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076447.729591:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076447.729601:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076447.729606:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076447/real 1769076447] req@ffff9350103f9180 x1855008913293184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076447.729622:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913293568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076447.729635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076447.729639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076447.729642:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076447.729652:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2153 00000100:00100000:2.0:1769076447.729655:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913293184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076447.729657:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076447.729668:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076447.729678:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076447.729694:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076447.729696:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913293696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076447.729711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076447.729714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076447.729722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769076447.729726:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913293184:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076447.729727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076447.729739:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2153 00000100:00080000:0.0:1769076447.729743:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076447.729748:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076447.729753:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076447.729764:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913293824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076447.729770:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913293312:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076447.729795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076447.729800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076447.729803:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076447.729808:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2153 00000100:00080000:0.0:1769076447.729814:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076447.729820:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076447.729824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:0.0:1769076447.729827:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076447.729843:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913293952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076447.729857:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076447/real 1769076447] req@ffff9350103faa00 x1855008913293312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076447.729882:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076447.729886:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913293312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076447.729913:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076447.729919:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076447.729924:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913293312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076447.729936:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913293440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076447.729961:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076447.729973:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076447/real 1769076447] req@ffff9350103f8700 x1855008913293440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076447.729986:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913293440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076447.729995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076447.729998:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076447.730002:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913293440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076447.730007:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913293568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076447.730022:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076447.730030:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076447/real 1769076447] req@ffff9350103f8380 x1855008913293568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076447.730041:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913293568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076447.730080:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076447.730084:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076447.730088:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913293568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076447.730094:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293696:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076447.730114:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076447.730125:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076447/real 1769076447] req@ffff9350103fb100 x1855008913293696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076447.730138:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913293696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076447.730164:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076447.730168:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076447.730183:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293696:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076447.730205:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076447.730225:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076447.730237:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076447/real 1769076447] req@ffff9350103f9180 x1855008913293824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076447.730259:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913293824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076447.730267:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076447.730270:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076447.730274:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076447.730279:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076447.730294:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076447.730304:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076447/real 1769076447] req@ffff9350103fb480 x1855008913293952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076447.730316:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913293952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076447.730335:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076447.730339:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076447.730343:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913293952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076447.921131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076448.433126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076448.433127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076448.433139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076448.433228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076448.753198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076448.945202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076449.457111:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076449.457133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076449.457134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076449.457212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076449.777175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076449.969163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076450.481124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076450.481133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076450.481152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076450.481188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076450.801117:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076450.993135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076451.505127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076451.505129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076451.505139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076451.505275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076451.825151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076452.017163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076452.529272:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076452.529289:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076452.529331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076452.529434:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:0.0:1769076452.849129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076452.849143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076452.849148:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076452.849149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076452.849159:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2158 00000100:00080000:0.0:1769076452.849171:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076452.849180:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076452.849232:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913294080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076452.849255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076452.849259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076452.849261:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076452.849265:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2158 00000100:00080000:0.0:1769076452.849270:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076452.849274:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076452.849285:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076452.849289:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913294080:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076452.849313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913294208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076452.849323:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076452.849327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076452.849329:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076452.849332:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2158 00000100:00080000:0.0:1769076452.849336:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076452.849339:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076452.849345:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076452.849355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913294336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076452.849363:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076452.849367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076452.849368:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076452.849373:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2158 00000100:00080000:0.0:1769076452.849377:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076452.849381:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076452.849385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076452.849395:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913294464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076452.849406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076452.849410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076452.849412:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076452.849412:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913294208:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076452.849418:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2158 00000100:00080000:0.0:1769076452.849421:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076452.849426:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076452.849430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076452.849443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913294592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076452.849446:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913294336:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769076452.849454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076452.849458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076452.849460:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076452.849464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2158 00000100:00080000:0.0:1769076452.849468:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076452.849468:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913294464:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769076452.849472:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076452.849477:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076452.849488:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913294720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076452.849492:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913294592:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076452.849499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076452.849503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076452.849505:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076452.849509:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2158 00000100:00080000:0.0:1769076452.849512:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076452.849513:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913294720:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769076452.849516:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076452.849521:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076452.849534:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913294848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076452.849575:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913294848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076453.041205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076453.553148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076453.553148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076453.553155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076453.553172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076453.873353:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076454.065130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076454.577158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076454.577181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076454.577222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076454.577422:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076454.897141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076455.089161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076455.601155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076455.601155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076455.601161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076455.601199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076455.921205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076455.921594:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076455.921663:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076455.921691:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076455.921748:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076455.921760:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076455.921770:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076455.921779:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076455.921789:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076455.921798:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076455.921814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076452/real 1769076455] req@ffff9350103f9c00 x1855008913294720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076455.921841:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913294720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076455.921856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076455.921862:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076455.921869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913294720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076455.921879:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076452/real 1769076455] req@ffff9350103f9500 x1855008913294592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076455.921895:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913294592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076455.921905:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076455.921909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076455.921913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913294592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076455.921920:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076452/real 1769076455] req@ffff9350103f9f80 x1855008913294208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076455.921933:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913294208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076455.921942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076455.921946:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076455.921949:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913294208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076455.921956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076452/real 1769076455] req@ffff9350103fb800 x1855008913294336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076455.921968:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913294336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076455.921976:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076455.921980:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076455.921984:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913294336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076455.921991:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076452/real 1769076455] req@ffff9350103fad80 x1855008913294464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076455.922005:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913294464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076455.922014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076455.922017:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076455.922021:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913294464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076455.922030:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076452/real 1769076455] req@ffff9350103fbb80 x1855008913294080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076455.922045:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913294080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076455.922076:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076455.922080:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076455.922085:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913294080:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076455.922132:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076452/real 1769076455] req@ffff9350103fb480 x1855008913294848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076455.922148:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913294848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076455.922156:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076455.922160:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076455.922165:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913294848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076456.113151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076456.625162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076456.625163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076456.625164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076456.625220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076456.945178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076457.137190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076457.649124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076457.649125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076457.649136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076457.649228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076457.969191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.969226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076457.969233:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.969248:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2163 00000100:00100000:2.0:1769076457.969256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076457.969266:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.969281:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076457.969336:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913294976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.969372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.969376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076457.969379:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.969384:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2163 00000100:00080000:0.0:1769076457.969389:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.969394:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076457.969426:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076457.969439:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913294976:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076457.969451:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913295104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.969468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.969473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076457.969476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.969482:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2163 00000100:00080000:0.0:1769076457.969487:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.969493:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076457.969503:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076457.969517:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913295232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.969531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.969536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076457.969538:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.969542:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2163 00000100:00080000:0.0:1769076457.969547:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076457.969555:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076457.969590:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076457.969599:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076457.969615:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913295360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.969627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.969631:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076457.969636:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076457.969636:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9350103fb480 x1855008913294976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076457.969650:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2163 00000100:00080000:0.0:1769076457.969661:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.969669:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076457.969675:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076457.969676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913294976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.969692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076457.969693:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913295488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076457.969703:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076457.969712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.969717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076457.969723:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076457.969729:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913294976:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076457.969732:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2163 00000100:00080000:0.0:1769076457.969740:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.969746:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076457.969757:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076457.969769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913295616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076457.969780:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913295104:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076457.969787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.969811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076457.969815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.969823:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2163 00000800:00000100:2.0:1769076457.969830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076457.969834:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.969847:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076457.969857:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076457.969859:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9350103fbb80 x1855008913295104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076457.969883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913295744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076457.969909:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913295104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.969930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.969938:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076457.969945:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913295104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076457.969957:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913295232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076457.969996:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076457.970020:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9350103fad80 x1855008913295232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.970115:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913295232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.970132:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.970138:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076457.970145:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913295232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076457.970156:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913295360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076457.970192:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076457.970242:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9350103fb800 x1855008913295360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.970267:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913295360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.970280:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.970286:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076457.970292:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913295360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076457.970303:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913295488:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076457.970339:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076457.970379:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9350103f9f80 x1855008913295488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.970401:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913295488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.970415:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.970420:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076457.970428:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913295488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076457.970469:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913295616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076457.970504:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076457.970524:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9350103f9500 x1855008913295616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.970545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913295616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.970559:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.970564:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076457.970569:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913295616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076457.970579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913295744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076457.970617:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076457.970635:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9350103f9c00 x1855008913295744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.970654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913295744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.970668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.970674:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076457.970680:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913295744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076458.161162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076458.673189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076458.673228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076458.673243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076458.673331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076458.993232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076459.185183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076459.697171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076459.697230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076459.697327:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076459.697367:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076460.017148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076460.209133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076460.721182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076460.721197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076460.721296:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076460.721335:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076461.041209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076461.233236:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076461.745214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076461.745288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076461.745292:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076461.745337:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076462.065158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076462.257161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076462.769208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076462.769210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076462.769311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076462.769319:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076463.089267:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076463.089330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076463.089341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076463.089356:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076463.089387:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2168 00000100:00080000:1.0:1769076463.089402:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076463.089411:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076463.089464:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913295872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076463.089503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076463.089511:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076463.089514:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076463.089519:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2168 00000100:00080000:1.0:1769076463.089523:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076463.089527:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076463.089540:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076463.089553:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913296000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076463.089565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076463.089569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076463.089571:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076463.089575:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2168 00000100:00080000:1.0:1769076463.089578:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076463.089600:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076463.089606:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076463.089620:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913296128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076463.089632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076463.089636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076463.089639:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076463.089644:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2168 00000100:00080000:1.0:1769076463.089648:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076463.089653:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076463.089660:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076463.089674:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913295872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076463.089678:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913296256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076463.089703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076463.089708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076463.089710:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076463.089715:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2168 00000100:00080000:1.0:1769076463.089720:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076463.089724:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076463.089730:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076463.089744:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913296384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076463.089757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076463.089761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076463.089764:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076463.089768:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2168 00000100:00080000:1.0:1769076463.089773:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076463.089776:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076463.089781:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076463.089795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913296512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076463.089806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076463.089814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076463.089816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076463.089824:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076463.089831:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2168 00000100:00080000:1.0:1769076463.089838:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076463.089844:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076463.089851:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076463.089866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913296640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076463.089894:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076463/real 1769076463] req@ffff9350100c8380 x1855008913295872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076463.089925:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913295872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076463.089947:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076463.089956:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076463.089964:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913295872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076463.089979:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296000:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076463.090026:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076463.090156:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076463/real 1769076463] req@ffff9350100ca300 x1855008913296000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076463.090178:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913296000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076463.090190:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076463.090194:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076463.090198:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076463.090207:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076463.090237:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076463.090251:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076463/real 1769076463] req@ffff9350100ca680 x1855008913296128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076463.090265:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913296128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076463.090274:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076463.090278:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076463.090282:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076463.090314:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076463.090332:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076463.090342:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076463/real 1769076463] req@ffff9350100c8e00 x1855008913296256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076463.090355:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913296256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076463.090366:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076463.090370:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076463.090374:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076463.090379:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913296384:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076463.090395:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076463.090405:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076463/real 1769076463] req@ffff9350100c9880 x1855008913296384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076463.090417:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913296384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076463.090427:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076463.090431:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076463.090434:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913296384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076463.090440:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913296512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076463.090456:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076463.090465:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076463/real 1769076463] req@ffff9350100c9500 x1855008913296512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076463.090477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913296512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076463.090486:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076463.090490:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076463.090494:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913296512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076463.090500:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913296640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076463.090536:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076463.090547:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076463/real 1769076463] req@ffff9350100caa00 x1855008913296640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076463.090561:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913296640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076463.090570:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076463.090574:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076463.090577:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913296640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076463.281230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076463.793248:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076463.793257:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076463.793298:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076463.793469:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076464.113270:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076464.305201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076464.817200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076464.817242:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076464.817372:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076464.817387:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076465.137227:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076465.329158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076465.841158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076465.841201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076465.841209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076465.841332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076466.161157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076466.353150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076466.865200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076466.865206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076466.865288:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076466.865293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076467.185191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076467.377177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076467.889215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076467.889237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076467.889257:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076467.889262:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:1.0:1769076468.209176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076468.209190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1769076468.209196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076468.209196:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076468.209212:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2173 00000100:00080000:1.0:1769076468.209229:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076468.209240:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076468.209291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913296768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076468.209330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076468.209335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076468.209338:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076468.209343:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2173 00000100:00080000:1.0:1769076468.209347:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076468.209351:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076468.209364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076468.209385:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913296896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076468.209401:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913296768:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076468.209417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076468.209425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076468.209429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076468.209439:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2173 00000100:00080000:1.0:1769076468.209446:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076468.209454:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076468.209465:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076468.209490:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913297024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076468.209511:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076468.209513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076468.209523:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076468.209527:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076468.209538:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2173 00000100:00080000:1.0:1769076468.209546:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076468.209552:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076468.209559:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076468.209572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076468/real 1769076468] req@ffff9350100cb800 x1855008913296768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076468.209583:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913297152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076468.209618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076468.209623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076468.209626:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076468.209630:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913296768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076468.209637:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2173 00000100:00080000:1.0:1769076468.209648:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076468.209659:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076468.209667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769076468.209668:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076468.209682:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769076468.209693:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913297280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076468.209694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913296768:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076468.209730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076468.209735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076468.209737:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076468.209742:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2173 00000100:00080000:1.0:1769076468.209748:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076468.209754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296896:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076468.209756:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076468.209772:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076468.209791:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913297408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076468.209805:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076468.209806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076468.209814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076468.209817:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076468.209825:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2173 00000100:00080000:1.0:1769076468.209830:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076468.209832:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076468/real 1769076468] req@ffff9350100caa00 x1855008913296896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076468.209841:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076468.209853:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076468.209867:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913296896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076468.209874:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913297536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076468.209897:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076468.209909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076468.209916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913296896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076468.209925:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076468.209948:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076468.209959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076468/real 1769076468] req@ffff9350100c9500 x1855008913297024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076468.209973:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913297024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076468.209983:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076468.209986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076468.209990:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076468.209996:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076468.210012:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076468.210021:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076468/real 1769076468] req@ffff9350100c9880 x1855008913297152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076468.210112:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913297152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076468.210125:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076468.210129:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076468.210133:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076468.210141:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913297280:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076468.210188:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076468.210216:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076468/real 1769076468] req@ffff9350100c8e00 x1855008913297280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076468.210231:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913297280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076468.210241:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076468.210245:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076468.210249:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913297280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076468.210270:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913297408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076468.210291:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076468.210301:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076468/real 1769076468] req@ffff9350100ca680 x1855008913297408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076468.210315:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913297408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076468.210332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076468.210336:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076468.210340:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913297408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076468.210346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913297536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076468.210361:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076468.210371:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076468/real 1769076468] req@ffff9350100ca300 x1855008913297536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076468.210383:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913297536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076468.210393:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076468.210396:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076468.210399:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913297536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076468.401162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076468.913277:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076468.913290:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076468.913344:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076468.913445:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076469.233279:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076469.425140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076469.937171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076469.937224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076469.937296:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076469.937338:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076470.257216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076470.449146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076470.961192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076470.961262:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076470.961290:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076470.961322:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076471.281242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076471.473151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076471.985151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076471.985227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076471.985244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076471.985249:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076472.305117:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076472.497115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076473.009220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076473.009228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076473.009239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076473.009285:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076473.329168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076473.329183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076473.329195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076473.329202:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076473.329215:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2179 00000100:00080000:1.0:1769076473.329232:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076473.329243:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076473.329308:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913297664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076473.329346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076473.329352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076473.329355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076473.329361:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2179 00000100:00080000:1.0:1769076473.329365:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076473.329370:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076473.329381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076473.329396:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913297792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076473.329409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076473.329414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076473.329418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076473.329424:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2179 00000100:00080000:1.0:1769076473.329428:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076473.329433:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076473.329440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076473.329459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913297920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076473.329460:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913297664:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076473.329487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076473.329491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076473.329493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076473.329498:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2179 00000100:00080000:1.0:1769076473.329502:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076473.329506:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076473.329512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076473.329528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913298048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076473.329541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076473.329546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076473.329548:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076473.329552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2179 00000100:00080000:1.0:1769076473.329557:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076473.329562:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076473.329576:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076473.329592:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076473.329596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913298176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076473.329612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076473.329617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076473.329620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076473.329626:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2179 00000100:00080000:1.0:1769076473.329630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076473.329635:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076473.329642:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076473.329658:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076473/real 1769076473] req@ffff9350100c8380 x1855008913297664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076473.329665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913298304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076473.329711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076473.329716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076473.329718:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076473.329725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913297664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076473.329731:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2179 00000100:00080000:1.0:1769076473.329743:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076473.329755:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076473.329768:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076473.329779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076473.329789:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913298432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076473.329795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076473.329813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913297664:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076473.329833:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297792:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076473.329866:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076473.329882:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076473/real 1769076473] req@ffff9350100c9f80 x1855008913297792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076473.329904:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913297792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076473.329922:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076473.329929:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076473.329935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076473.330025:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076473.330115:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076473.330133:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076473/real 1769076473] req@ffff9350100c9c00 x1855008913297920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076473.330150:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913297920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076473.330161:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076473.330166:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076473.330170:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913297920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076473.330179:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076473.330200:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076473.330211:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076473/real 1769076473] req@ffff9350100c8a80 x1855008913298048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076473.330225:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913298048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076473.330235:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076473.330238:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076473.330242:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076473.330249:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913298176:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076473.330267:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076473.330277:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076473/real 1769076473] req@ffff9350100c8000 x1855008913298176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076473.330291:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913298176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076473.330312:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076473.330315:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076473.330319:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913298176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076473.330325:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913298304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076473.330345:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076473.330366:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076473/real 1769076473] req@ffff9350100cb100 x1855008913298304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076473.330391:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913298304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076473.330408:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076473.330414:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076473.330421:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913298304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076473.330432:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913298432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076473.330471:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076473.330490:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076473/real 1769076473] req@ffff9350100c9180 x1855008913298432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076473.330506:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913298432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076473.330516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076473.330520:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076473.330524:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913298432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076473.521151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076474.033186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076474.033192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076474.033196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076474.033312:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076474.353211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076474.545155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076475.057216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076475.057231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076475.057233:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076475.057281:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076475.377176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076475.569190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076476.081185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076476.081193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076476.081220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076476.081322:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076476.401160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076476.593174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076477.105265:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076477.105279:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076477.105298:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076477.105380:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076477.425127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076477.617157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076478.129191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076478.129218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076478.129308:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076478.129341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076478.449165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076478.449250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076478.449261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076478.449267:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076478.449280:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2184 00000100:00080000:3.0:1769076478.449296:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076478.449307:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076478.449358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913298560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076478.449401:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076478.449406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076478.449408:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076478.449414:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2184 00000100:00080000:3.0:1769076478.449418:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076478.449423:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076478.449436:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076478.449453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913298688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076478.449467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076478.449472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076478.449475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076478.449480:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2184 00000100:00080000:3.0:1769076478.449484:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076478.449487:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076478.449495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076478.449509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913298816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076478.449529:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913298560:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076478.449534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076478.449558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076478.449562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076478.449571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2184 00000100:00080000:3.0:1769076478.449578:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076478.449584:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076478.449592:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076478.449609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913298944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076478.449622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076478.449627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076478.449630:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076478.449635:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2184 00000100:00080000:3.0:1769076478.449639:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076478.449643:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076478.449656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076478.449662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076478.449689:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913299072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076478.449703:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076478.449708:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076478.449711:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076478.449715:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2184 00000100:00080000:3.0:1769076478.449722:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076478.449730:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076478.449739:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076478.449758:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076478/real 1769076478] req@ffff935002a7ca80 x1855008913298560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076478.449760:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913299200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076478.449830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076478.449849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076478.449852:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076478.449886:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2184 00000100:00080000:3.0:1769076478.449895:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076478.449903:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913298560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076478.449909:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076478.449933:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076478.449947:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076478.449959:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769076478.449960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913299328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076478.449977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913298560:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076478.450022:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298688:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076478.450139:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076478.450159:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076478/real 1769076478] req@ffff935002a7e300 x1855008913298688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076478.450176:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913298688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076478.450188:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076478.450193:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076478.450198:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076478.450238:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076478.450262:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076478.450274:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076478/real 1769076478] req@ffff935002a7f100 x1855008913298816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076478.450301:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913298816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076478.450317:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076478.450323:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076478.450330:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076478.450343:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076478.450380:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076478.450401:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076478/real 1769076478] req@ffff935002a7c380 x1855008913298944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076478.450426:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913298944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076478.450441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076478.450447:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076478.450453:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913298944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076478.450465:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299072:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076478.450500:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076478.450521:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076478/real 1769076478] req@ffff935002a7fb80 x1855008913299072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076478.450545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913299072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076478.450575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076478.450583:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076478.450590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299072:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076478.450602:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076478.450631:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076478.450646:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076478/real 1769076478] req@ffff935002a7c700 x1855008913299200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076478.450661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913299200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076478.450671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076478.450676:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076478.450680:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076478.450690:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076478.450724:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076478.450745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076478/real 1769076478] req@ffff935002a7dc00 x1855008913299328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076478.450770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913299328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076478.450787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076478.450794:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076478.450802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076478.641175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076479.153217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076479.153218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076479.153218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076479.153298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076479.473224:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076479.665212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076480.177176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076480.177192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076480.177204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076480.177314:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076480.497163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076480.689146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076481.201145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076481.201228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076481.201234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076481.201318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076481.521143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076481.713177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076482.225189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076482.225208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076482.225232:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076482.225326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076482.545176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076482.737160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076483.249194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076483.249269:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076483.249275:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076483.249324:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076483.569148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076483.569232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076483.569243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076483.569249:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076483.569261:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2189 00000100:00080000:2.0:1769076483.569291:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076483.569302:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076483.569356:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913299456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076483.569399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076483.569404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076483.569407:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076483.569412:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2189 00000100:00080000:2.0:1769076483.569417:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076483.569421:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076483.569434:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076483.569450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913299584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076483.569465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076483.569478:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913299456:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076483.569478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076483.569483:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076483.569498:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2189 00000100:00080000:2.0:1769076483.569502:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076483.569506:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076483.569512:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076483.569527:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913299712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076483.569540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076483.569544:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076483.569547:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076483.569551:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2189 00000100:00080000:2.0:1769076483.569556:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076483.569560:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076483.569566:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076483.569578:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913299840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076483.569590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076483.569594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076483.569597:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076483.569606:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2189 00000800:00000100:0.0:1769076483.569610:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076483.569613:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076483.569621:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076483.569630:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076483.569646:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913299968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076483.569658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076483.569663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076483.569666:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076483.569672:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2189 00000100:00100000:0.0:1769076483.569680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076483/real 1769076483] req@ffff935002a60a80 x1855008913299456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076483.569681:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076483.569693:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076483.569705:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076483.569731:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913299456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076483.569732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913300096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076483.569817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076483.569823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076483.569824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076483.569830:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076483.569845:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2189 00000100:00080000:0.0:1769076483.569847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076483.569851:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076483.569862:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076483.569870:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913299456:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076483.569872:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076483.569894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913300224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076483.569926:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913299584:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076483.569968:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076483.569987:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076483/real 1769076483] req@ffff935002a60000 x1855008913299584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076483.570005:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913299584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076483.570017:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076483.570021:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076483.570027:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913299584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076483.570035:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913299712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076483.570095:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076483.570110:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076483/real 1769076483] req@ffff935002a62a00 x1855008913299712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076483.570125:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913299712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076483.570136:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076483.570140:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076483.570145:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913299712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076483.570152:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913299840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076483.570173:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076483.570184:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076483/real 1769076483] req@ffff935002a61880 x1855008913299840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076483.570199:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913299840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076483.570209:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076483.570213:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076483.570217:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913299840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076483.570224:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299968:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076483.570243:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076483.570256:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076483/real 1769076483] req@ffff935002a62680 x1855008913299968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076483.570270:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913299968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076483.570280:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076483.570284:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076483.570288:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913299968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076483.570294:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076483.570313:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076483.570325:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076483/real 1769076483] req@ffff935002a62300 x1855008913300096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076483.570339:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913300096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076483.570349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076483.570352:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076483.570356:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076483.570362:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076483.570379:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076483.570391:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076483/real 1769076483] req@ffff935002a61500 x1855008913300224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076483.570413:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913300224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076483.570430:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076483.570436:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076483.570443:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076483.761137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076484.273192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076484.273193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076484.273203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076484.273313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076484.593187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076484.785159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076485.297206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076485.297221:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076485.297224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076485.297320:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076485.617184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076485.809169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076486.321174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076486.321184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076486.321205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076486.321404:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076486.641179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076486.833162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076487.345170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076487.345181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076487.345181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076487.345413:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076487.665159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076487.857170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076488.369182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076488.369186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076488.369202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076488.369426:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076488.689201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076488.689223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076488.689239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076488.689248:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076488.689263:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2194 00000100:00080000:3.0:1769076488.689279:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076488.689292:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076488.689356:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913300352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076488.689414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076488.689419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076488.689422:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076488.689428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2194 00000100:00080000:3.0:1769076488.689434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076488.689440:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076488.689456:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076488.689477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913300480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076488.689490:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913300352:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076488.689500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076488.689507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076488.689510:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076488.689516:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2194 00000100:00080000:3.0:1769076488.689521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076488.689528:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076488.689536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076488.689551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913300608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076488.689564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076488.689570:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076488.689572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076488.689576:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2194 00000100:00080000:3.0:1769076488.689580:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076488.689585:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076488.689590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076488.689607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913300736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076488.689619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076488.689625:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076488.689626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076488.689633:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076488.689642:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2194 00000100:00080000:3.0:1769076488.689646:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076488.689651:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076488.689657:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076488.689714:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913300864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076488.689730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076488.689736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076488.689739:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076488.689745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076488/real 1769076488] req@ffff935002a7f800 x1855008913300352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076488.689749:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2194 00000100:00080000:3.0:1769076488.689761:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076488.689775:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076488.689786:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076488.689800:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913300352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076488.689809:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913300992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076488.689858:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076488.689860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076488.689863:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076488.689870:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076488.689880:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2194 00000100:00080000:0.0:1769076488.689883:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076488.689886:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076488.689897:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076488.689905:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076488.689907:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913300352:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076488.689921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913301120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076488.689964:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913300480:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076488.690005:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076488.690025:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076488/real 1769076488] req@ffff935002a7c000 x1855008913300480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076488.690090:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913300480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076488.690104:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076488.690109:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076488.690114:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913300480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076488.690123:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913300608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076488.690151:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076488.690166:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076488/real 1769076488] req@ffff935002a7df80 x1855008913300608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076488.690182:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913300608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076488.690193:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076488.690197:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076488.690202:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913300608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076488.690209:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913300736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076488.690228:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076488.690239:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076488/real 1769076488] req@ffff935002a7d180 x1855008913300736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076488.690253:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913300736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076488.690265:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076488.690269:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076488.690273:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913300736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076488.690280:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076488.690297:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076488.690308:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076488/real 1769076488] req@ffff935002a7dc00 x1855008913300864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076488.690322:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913300864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076488.690333:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076488.690337:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076488.690340:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076488.690347:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076488.690364:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076488.690376:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076488/real 1769076488] req@ffff935002a7c700 x1855008913300992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076488.690389:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913300992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076488.690399:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076488.690402:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076488.690407:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913300992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076488.690415:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913301120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076488.690433:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076488.690444:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076488/real 1769076488] req@ffff935002a7fb80 x1855008913301120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076488.690458:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913301120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076488.690468:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076488.690472:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076488.690475:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913301120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076488.881152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076489.393184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076489.393190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076489.393191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076489.393278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076489.713210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076489.905144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076490.417124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076490.417133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076490.417172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076490.417228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076490.737122:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076490.929166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076491.441158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076491.441169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076491.441190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076491.441195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076491.761211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076491.953159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076492.465182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076492.465235:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076492.465247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076492.465261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076492.785269:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076492.977121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076493.489168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076493.489245:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076493.489250:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076493.489314:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076493.809115:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076493.809127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076493.809136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076493.809142:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076493.809152:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2199 00000100:00080000:3.0:1769076493.809165:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076493.809174:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076493.809217:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913301248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076493.809248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076493.809252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076493.809255:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076493.809261:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2199 00000100:00080000:3.0:1769076493.809266:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076493.809271:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076493.809279:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913301248:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076493.809283:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076493.809301:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913301376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076493.809314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076493.809318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076493.809320:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076493.809325:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2199 00000100:00080000:3.0:1769076493.809330:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076493.809335:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076493.809341:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076493.809355:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076493.809356:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913301504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076493.809369:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076493.809373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076493.809376:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076493.809381:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2199 00000100:00080000:3.0:1769076493.809384:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076493.809388:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076493.809393:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076493.809400:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076493/real 1769076493] req@ffff935002a7c380 x1855008913301248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076493.809407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913301632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076493.809422:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913301248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076493.809432:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076493.809437:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076493.809439:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076493.809444:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2199 00000100:00080000:3.0:1769076493.809450:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076493.809458:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076493.809466:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076493.809466:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076493.809476:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769076493.809485:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913301760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076493.809486:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913301248:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076493.809521:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076493.809526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076493.809529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076493.809530:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913301376:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076493.809540:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2199 00000100:00080000:3.0:1769076493.809547:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076493.809553:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076493.809559:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076493.809561:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076493.809572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076493/real 1769076493] req@ffff935002a7f100 x1855008913301376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076493.809590:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913301888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076493.809595:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913301376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076493.809630:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076493.809632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076493.809635:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076493.809640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076493.809641:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913301376:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076493.809644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076493.809650:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913301504:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076493.809652:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2199 00000100:00080000:3.0:1769076493.809659:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076493.809664:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076493.809672:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076493.809675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076493.809687:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076493/real 1769076493] req@ffff935002a7e300 x1855008913301504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076493.809692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913302016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076493.809703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913301504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076493.809716:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076493.809719:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076493.809722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913301504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076493.809728:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913301632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076493.809741:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076493.809748:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076493/real 1769076493] req@ffff935002a7ca80 x1855008913301632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076493.809756:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913301632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076493.809762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076493.809764:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076493.809767:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913301632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076493.809770:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913301760:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076493.809780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076493.809785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076493/real 1769076493] req@ffff935002a7fb80 x1855008913301760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076493.809792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913301760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076493.809798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076493.809800:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076493.809802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913301760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076493.809818:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913301888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076493.809829:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076493.809836:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076493/real 1769076493] req@ffff935002a7c700 x1855008913301888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076493.809844:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913301888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076493.809850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076493.809852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076493.809854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913301888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076493.809857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076493.809867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076493.809873:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076493/real 1769076493] req@ffff935002a7dc00 x1855008913302016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076493.809880:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913302016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076493.809886:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076493.809887:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076493.809889:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076494.001155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076494.513146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076494.513146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076494.513171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076494.513455:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076494.833126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076495.025155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076495.537143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076495.537147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076495.537206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076495.537211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076495.857179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076496.049156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076496.561136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076496.561143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076496.561180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076496.561189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076496.881175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076497.073123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076497.585162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076497.585162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076497.585206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076497.585244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076497.905221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076498.097150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076498.609135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076498.609136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076498.609144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076498.609176:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076498.929111:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076498.929120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.929128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076498.929132:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.929140:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2204 00000100:00080000:3.0:1769076498.929151:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.929160:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076498.929202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913302144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.929232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.929235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076498.929236:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.929240:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2204 00000100:00080000:3.0:1769076498.929243:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.929246:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076498.929256:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076498.929268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913302272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076498.929273:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913302144:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076498.929288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.929291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076498.929292:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.929296:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2204 00000100:00080000:3.0:1769076498.929299:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.929301:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076498.929304:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076498.929313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913302400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.929319:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.929322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076498.929323:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.929326:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2204 00000100:00080000:3.0:1769076498.929328:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.929331:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076498.929333:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076498.929340:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913302528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.929346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.929348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076498.929350:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.929353:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2204 00000800:00000100:0.0:1769076498.929358:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076498.929358:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.929364:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076498.929368:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076498.929378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913302656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.929385:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.929387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076498.929388:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.929391:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2204 00000100:00080000:3.0:1769076498.929395:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.929397:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076498.929401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076498.929402:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff935002a7d180 x1855008913302144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076498.929412:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913302784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.929419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076498.929423:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913302144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076498.929424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076498.929427:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.929435:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2204 00000100:00080000:3.0:1769076498.929440:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076498.929441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076498.929443:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076498.929448:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076498.929449:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076498.929456:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913302144:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076498.929460:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913302912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076498.929480:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913302272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076498.929504:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076498.929544:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff935002a7df80 x1855008913302272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076498.929561:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913302272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076498.929571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076498.929591:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076498.929595:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913302272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076498.929604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913302400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076498.929634:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076498.929648:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff935002a7c000 x1855008913302400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076498.929671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913302400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076498.929680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076498.929684:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076498.929688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913302400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076498.929710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913302528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076498.929721:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076498.929727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff935002a7f800 x1855008913302528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076498.929735:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913302528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076498.929740:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076498.929742:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076498.929743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913302528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076498.929747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302656:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076498.929757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076498.929763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff935002a7dc00 x1855008913302656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076498.929770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913302656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076498.929781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076498.929783:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076498.929785:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076498.929788:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076498.929797:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076498.929803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff935002a7c700 x1855008913302784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076498.929809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913302784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076498.929815:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076498.929817:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076498.929819:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076498.929822:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076498.929831:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076498.929836:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff935002a7fb80 x1855008913302912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076498.929843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913302912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076498.929849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076498.929850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076498.929853:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913302912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076499.121100:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076499.633158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076499.633159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076499.633159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076499.633239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076499.953200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076500.145142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076500.657138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076500.657157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076500.657166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076500.657262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076500.977155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076501.169125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076501.681134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076501.681143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076501.681273:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076501.681286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076502.001197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076502.193149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076502.705171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076502.705173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076502.705234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076502.705373:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076503.025193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076503.217144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076503.729141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076503.729141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076503.729143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076503.729249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076504.049177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076504.049189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076504.049198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076504.049203:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076504.049212:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2209 00000100:00080000:2.0:1769076504.049224:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076504.049231:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076504.049272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913303040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076504.049297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076504.049300:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076504.049302:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076504.049305:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2209 00000100:00080000:2.0:1769076504.049307:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076504.049310:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076504.049319:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076504.049330:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913303168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076504.049338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076504.049341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076504.049342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076504.049347:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2209 00000100:00080000:2.0:1769076504.049352:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076504.049353:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913303040:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076504.049357:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076504.049367:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076504.049378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913303296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076504.049384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076504.049387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076504.049388:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076504.049392:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2209 00000100:00080000:2.0:1769076504.049395:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076504.049398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076504.049401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076504.049409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913303424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076504.049415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076504.049417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076504.049418:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076504.049421:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2209 00000100:00080000:2.0:1769076504.049424:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076504.049427:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076504.049452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076504.049462:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913303552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076504.049469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076504.049478:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076504.049479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076504.049486:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076504.049492:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2209 00000100:00080000:2.0:1769076504.049495:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076504.049498:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076504.049502:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076504.049511:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913303680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076504.049518:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076504.049525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076504.049527:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076504.049530:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2209 00000100:00080000:2.0:1769076504.049535:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076504.049538:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076504.049541:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076504.049551:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076504/real 1769076504] req@ffff935002a61180 x1855008913303040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076504.049552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913303808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076504.049599:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913303040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076504.049612:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076504.049617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076504.049621:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913303040:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076504.049650:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913303168:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076504.049679:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076504.049695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076504/real 1769076504] req@ffff935002a61c00 x1855008913303168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076504.049709:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913303168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076504.049720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076504.049724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076504.049728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913303168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076504.049734:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913303296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076504.049752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076504.049764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076504/real 1769076504] req@ffff935002a61f80 x1855008913303296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076504.049777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913303296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076504.049786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076504.049790:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076504.049794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913303296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076504.049801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913303424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076504.049821:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076504.049833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076504/real 1769076504] req@ffff935002a63100 x1855008913303424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076504.049846:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913303424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076504.049856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076504.049860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076504.049864:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913303424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076504.049871:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913303552:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076504.049890:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076504.049902:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076504/real 1769076504] req@ffff935002a61500 x1855008913303552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076504.049914:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913303552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076504.049923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076504.049927:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076504.049931:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913303552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076504.049938:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913303680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076504.049959:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076504.049972:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076504/real 1769076504] req@ffff935002a62300 x1855008913303680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076504.049985:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913303680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076504.049993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076504.049996:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076504.050000:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913303680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076504.050007:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913303808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076504.050025:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076504.050036:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076504/real 1769076504] req@ffff935002a62680 x1855008913303808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076504.050072:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913303808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076504.050081:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076504.050084:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076504.050088:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913303808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076504.241116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076504.753145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076504.753145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076504.753151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076504.753270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076505.073163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076505.265129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076505.777114:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076505.777121:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076505.777125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076505.777229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076506.097156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076506.289106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076506.801138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076506.801138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076506.801138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076506.801254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076507.121174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076507.313122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076507.825112:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076507.825112:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076507.825117:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076507.825173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076508.145142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076508.337112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076508.849110:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076508.849114:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076508.849124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076508.849238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076509.169138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076509.169141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076509.169149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076509.169154:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076509.169162:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2214 00000100:00080000:3.0:1769076509.169173:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076509.169180:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076509.169217:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913303936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076509.169239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076509.169242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076509.169244:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076509.169247:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2214 00000100:00080000:3.0:1769076509.169250:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076509.169252:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076509.169260:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076509.169290:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913304064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076509.169302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076509.169310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076509.169311:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913303936:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076509.169313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076509.169326:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2214 00000100:00080000:3.0:1769076509.169365:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076509.169371:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076509.169377:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076509.169393:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913304192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076509.169418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076509.169421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076509.169423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076509.169426:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2214 00000100:00080000:3.0:1769076509.169430:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076509.169434:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076509.169438:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076509.169447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913304320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076509.169455:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076509.169466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076509.169486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076509.169487:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076509.169492:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2214 00000100:00080000:3.0:1769076509.169501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076509.169504:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076509.169508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076509.169520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913304448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076509.169527:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076509/real 1769076509] req@ffff935002a7ca80 x1855008913303936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076509.169531:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076509.169539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076509.169542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076509.169548:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2214 00000100:00080000:3.0:1769076509.169551:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076509.169553:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076509.169557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076509.169558:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913303936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076509.169568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913304576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076509.169588:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076509.169597:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076509.169604:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913303936:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076509.169610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076509.169615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076509.169618:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076509.169624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2214 00000100:00100000:0.0:1769076509.169628:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304064:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076509.169629:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076509.169638:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076509.169646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076509.169657:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076509.169660:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913304704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076509.169677:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076509/real 1769076509] req@ffff935002a7e300 x1855008913304064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076509.169695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913304064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076509.169705:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076509.169709:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076509.169711:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076509.169717:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076509.169733:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076509.169741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076509/real 1769076509] req@ffff935002a7f100 x1855008913304192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076509.169749:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913304192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076509.169755:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076509.169758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076509.169760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076509.169764:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076509.169774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076509.169780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076509/real 1769076509] req@ffff935002a7c380 x1855008913304320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076509.169787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913304320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076509.169793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076509.169795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076509.169797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076509.169800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913304448:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076509.169810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076509.169815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076509/real 1769076509] req@ffff935002a7fb80 x1855008913304448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076509.169823:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913304448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076509.169828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076509.169830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076509.169832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913304448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076509.169835:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913304576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076509.169845:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076509.169850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076509/real 1769076509] req@ffff935002a7c700 x1855008913304576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076509.169858:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913304576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076509.169863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076509.169865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076509.169867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913304576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076509.169883:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913304704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076509.169901:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076509.169912:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076509/real 1769076509] req@ffff935002a7dc00 x1855008913304704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076509.169926:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913304704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076509.169943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076509.169948:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076509.169952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913304704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076509.361117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076509.873121:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076509.873122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076509.873128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076509.873181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076510.193199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076510.385154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076510.897161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076510.897165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076510.897212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076510.897228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076511.217202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076511.409112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076511.921116:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076511.921129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076511.921134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076511.921197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076512.241116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076512.433111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076512.945137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076512.945137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076512.945169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076512.945202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076513.265124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076513.457172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076513.969121:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076513.969126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076513.969148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076513.969174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076514.289121:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076514.289123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076514.289132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076514.289137:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076514.289147:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2219 00000100:00080000:3.0:1769076514.289157:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076514.289164:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076514.289201:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913304832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076514.289225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076514.289228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076514.289229:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076514.289232:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2219 00000100:00080000:3.0:1769076514.289234:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076514.289237:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076514.289254:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076514.289257:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913304832:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076514.289279:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913304960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076514.289288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076514.289290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076514.289292:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076514.289295:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2219 00000100:00080000:3.0:1769076514.289298:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076514.289300:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076514.289305:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076514.289313:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913305088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076514.289320:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076514.289323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076514.289324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076514.289327:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2219 00000100:00080000:3.0:1769076514.289330:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076514.289333:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076514.289336:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076514.289338:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076514.289346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913305216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076514.289353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076514.289356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076514.289358:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076514.289361:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2219 00000100:00080000:3.0:1769076514.289363:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076514.289365:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076514.289368:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076514.289377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913305344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076514.289382:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076514/real 1769076514] req@ffff935002a7f800 x1855008913304832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076514.289391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076514.289399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076514.289402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076514.289406:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2219 00000100:00080000:3.0:1769076514.289409:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076514.289411:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913304832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076514.289415:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076514.289421:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076514.289428:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076514.289432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913305472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076514.289435:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076514.289443:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913304832:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076514.289451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076514.289455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076514.289456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076514.289459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2219 00000100:00080000:3.0:1769076514.289461:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076514.289464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076514.289467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076514.289474:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076514.289475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913305600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076514.289497:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076514.289509:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076514/real 1769076514] req@ffff935002a7c000 x1855008913304960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076514.289517:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913304960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076514.289523:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076514.289526:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076514.289528:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913304960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076514.289532:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076514.289543:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076514.289549:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076514/real 1769076514] req@ffff935002a7df80 x1855008913305088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076514.289571:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913305088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076514.289577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076514.289579:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076514.289581:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076514.289585:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076514.289596:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076514.289602:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076514/real 1769076514] req@ffff935002a7d180 x1855008913305216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076514.289609:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913305216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076514.289618:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076514.289621:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076514.289624:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076514.289637:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913305344:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076514.289658:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076514.289669:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076514/real 1769076514] req@ffff935002a7dc00 x1855008913305344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076514.289682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913305344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076514.289693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076514.289695:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076514.289697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913305344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076514.289701:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913305472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076514.289716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076514.289722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076514/real 1769076514] req@ffff935002a7c700 x1855008913305472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076514.289730:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913305472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076514.289735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076514.289737:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076514.289739:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913305472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076514.289753:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913305600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076514.289763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076514.289768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076514/real 1769076514] req@ffff935002a7fb80 x1855008913305600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076514.289775:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913305600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076514.289780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076514.289782:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076514.289784:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913305600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076514.481116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076514.993139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076514.993140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076514.993160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076514.993235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076515.313176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076515.505165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076516.017125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076516.017152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076516.017155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076516.017218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076516.337132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076516.529132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076517.041146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076517.041146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076517.041148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076517.041251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076517.361130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076517.553109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076518.065116:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076518.065123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076518.065131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076518.065171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076518.385187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076518.577208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076519.089138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076519.089161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076519.089220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076519.089239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076519.409208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076519.409247:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076519.409260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076519.409267:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076519.409294:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2225 00000100:00080000:1.0:1769076519.409314:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076519.409325:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076519.409399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913305728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076519.409434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076519.409441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076519.409460:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076519.409467:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2225 00000100:00080000:1.0:1769076519.409472:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076519.409479:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076519.409492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076519.409500:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913305728:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076519.409515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913305856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076519.409530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076519.409534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076519.409537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076519.409542:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2225 00000100:00080000:1.0:1769076519.409547:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076519.409552:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076519.409559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076519.409573:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913305984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076519.409585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076519.409589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076519.409591:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076519.409597:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2225 00000100:00080000:1.0:1769076519.409618:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076519.409623:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076519.409631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076519.409646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913306112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076519.409662:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076519.409669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769076519.409671:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305856:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076519.409673:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076519.409682:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2225 00000100:00080000:1.0:1769076519.409688:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076519.409694:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076519.409703:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076519.409741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769076519.409741:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913306240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076519.409760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076519.409767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076519.409771:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076519.409778:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2225 00000100:00080000:1.0:1769076519.409785:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076519.409808:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076519.409815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913306112:192.168.204.121@tcp:8:kworker.0 00000008:00100000:1.0:1769076519.409815:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076519.409830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913306368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076519.409844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076519.409849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076519.409851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913306240:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076519.409852:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076519.409874:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2225 00000100:00080000:1.0:1769076519.409882:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076519.409889:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076519.409898:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076519.409905:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913306368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769076519.409916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913306496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076519.409940:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913306496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076519.601141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076520.113267:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076520.113296:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076520.113296:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076520.113371:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076520.433242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076520.625234:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076521.137173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076521.137196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076521.137237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076521.137302:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076521.457191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076521.649202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076522.161188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076522.161190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076522.161241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076522.161394:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076522.481218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076522.481615:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076522.481696:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076522.481730:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076522.481804:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076522.481825:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076522.481840:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076522.481858:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076522.481883:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076522.481916:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076522.481939:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076519/real 1769076522] req@ffff9350100c8a80 x1855008913306240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076522.481989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913306240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076522.482007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076522.482013:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076522.482019:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913306240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076522.482029:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076519/real 1769076522] req@ffff9350100c9c00 x1855008913306368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076522.482090:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913306368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076522.482111:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076522.482156:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076522.482165:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913306368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076522.482183:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076519/real 1769076522] req@ffff9350100c9180 x1855008913305856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076522.482211:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913305856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076522.482228:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076522.482236:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076522.482242:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076522.482259:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076519/real 1769076522] req@ffff9350100cb100 x1855008913305984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076522.482281:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913305984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076522.482296:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076522.482302:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076522.482308:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913305984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076522.482322:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076519/real 1769076522] req@ffff9350100c8000 x1855008913306112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076522.482344:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913306112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076522.482359:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076522.482366:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076522.482371:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913306112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076522.482385:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076519/real 1769076522] req@ffff9350100cad80 x1855008913305728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076522.482410:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913305728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076522.482426:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076522.482432:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076522.482438:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913305728:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076522.482504:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076519/real 1769076522] req@ffff9350100c9f80 x1855008913306496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076522.482533:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913306496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076522.482551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076522.482560:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076522.482580:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913306496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076522.673197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076523.185168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076523.185171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076523.185178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076523.185308:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076523.505210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076523.697186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076524.209216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076524.209235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076524.209288:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076524.209323:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076524.529201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076524.529340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076524.529354:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076524.529361:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076524.529375:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2230 00000100:00080000:2.0:1769076524.529393:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076524.529405:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076524.529462:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913306624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076524.529505:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076524.529524:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076524.529529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076524.529538:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2230 00000100:00080000:2.0:1769076524.529547:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076524.529556:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076524.529572:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913306624:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076524.529575:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076524.529609:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913306752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076524.529627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076524.529632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076524.529635:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076524.529640:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2230 00000100:00080000:2.0:1769076524.529647:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076524.529653:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076524.529660:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076524.529675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076524.529681:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913306880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076524.529704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076524.529711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076524.529715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076524.529722:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2230 00000100:00080000:2.0:1769076524.529730:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076524.529738:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076524.529751:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076524.529754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076524/real 1769076524] req@ffff935002a61880 x1855008913306624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076524.529781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913307008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076524.529797:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913306624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076524.529804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076524.529822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076524.529829:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076524.529840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076524.529843:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2230 00000100:00080000:2.0:1769076524.529874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076524.529877:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076524.529889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076524.529898:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913306624:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076524.529906:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076524.529954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913307136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076524.529967:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913306752:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076524.529988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076524.529998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076524.530002:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076524.530011:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2230 00000100:00080000:2.0:1769076524.530020:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076524.530028:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076524.530038:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769076524.530045:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076524.530071:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076524/real 1769076524] req@ffff935002a62a00 x1855008913306752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076524.530148:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913307264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076524.530168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076524.530174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076524.530182:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076524.530199:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2230 00000100:00080000:2.0:1769076524.530212:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076524.530224:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913306752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076524.530225:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076524.530242:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076524.530254:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076524.530267:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076524.530280:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913307392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076524.530281:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913306752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076524.530315:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913306880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076524.530368:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076524.530393:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076524/real 1769076524] req@ffff935002a60000 x1855008913306880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076524.530419:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913306880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076524.530435:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076524.530442:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076524.530449:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913306880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076524.530461:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076524.530500:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076524.530524:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076524/real 1769076524] req@ffff935002a60a80 x1855008913307008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076524.530548:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913307008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076524.530566:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076524.530573:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076524.530578:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076524.530611:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913307136:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076524.530650:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076524.530671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076524/real 1769076524] req@ffff935002a62680 x1855008913307136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076524.530692:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913307136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076524.530708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076524.530714:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076524.530720:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913307136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076524.530732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913307264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076524.530772:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076524.530794:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076524/real 1769076524] req@ffff935002a62300 x1855008913307264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076524.530816:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913307264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076524.530829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076524.530835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076524.530842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913307264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076524.530852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913307392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076524.530886:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076524.530908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076524/real 1769076524] req@ffff935002a61500 x1855008913307392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076524.530932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913307392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076524.530948:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076524.530955:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076524.530961:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913307392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076524.721289:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076525.233142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076525.233181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076525.233195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076525.233346:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076525.553263:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076525.745157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076526.257193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076526.257193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076526.257198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076526.257371:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076526.577192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076526.769158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076527.281162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076527.281176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076527.281176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076527.281248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076527.601191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076527.793163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076528.305133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076528.305138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076528.305212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076528.305278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076528.625187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076528.817148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076529.329210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076529.329223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076529.329274:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076529.329289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076529.649197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076529.649234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076529.649243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076529.649248:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076529.649259:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2235 00000100:00080000:1.0:1769076529.649275:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076529.649283:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076529.649331:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913307520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076529.649362:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076529.649367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076529.649369:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076529.649373:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2235 00000100:00080000:1.0:1769076529.649377:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076529.649380:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076529.649390:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076529.649407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913307648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076529.649439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076529.649447:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913307520:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076529.649447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076529.649452:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076529.649466:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2235 00000100:00080000:1.0:1769076529.649470:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076529.649474:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076529.649481:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076529.649496:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913307776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076529.649506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076529.649510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076529.649513:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076529.649517:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2235 00000100:00080000:1.0:1769076529.649521:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076529.649525:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076529.649529:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076529.649542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913307904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076529.649553:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076529.649557:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076529.649559:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076529.649564:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2235 00000100:00080000:1.0:1769076529.649568:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076529.649576:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076529.649581:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769076529.649585:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076529.649601:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913308032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076529.649642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076529.649648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076529.649651:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076529.649658:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2235 00000100:00080000:1.0:1769076529.649665:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076529.649672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076529/real 1769076529] req@ffff9350100c8380 x1855008913307520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076529.649678:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076529.649692:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076529.649716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913307520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076529.649720:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913308160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076529.649782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076529.649784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076529.649796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076529.649799:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076529.649801:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076529.649829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913307520:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076529.649834:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2235 00000100:00080000:1.0:1769076529.649848:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076529.649858:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076529.649886:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076529.649896:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076529.649916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913308288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076529.649955:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076529.649987:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076529/real 1769076529] req@ffff9350100ca300 x1855008913307648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076529.650114:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913307648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076529.650128:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076529.650134:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076529.650139:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076529.650149:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076529.650180:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076529.650197:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076529/real 1769076529] req@ffff9350100ca680 x1855008913307776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076529.650215:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913307776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076529.650235:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076529.650239:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076529.650244:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076529.650252:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076529.650274:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076529.650287:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076529/real 1769076529] req@ffff9350100c8e00 x1855008913307904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076529.650303:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913307904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076529.650314:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076529.650318:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076529.650323:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913307904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076529.650330:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076529.650351:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076529.650364:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076529/real 1769076529] req@ffff9350100c9880 x1855008913308032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076529.650382:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913308032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076529.650394:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076529.650399:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076529.650405:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076529.650414:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076529.650459:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076529.650479:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076529/real 1769076529] req@ffff9350100c9500 x1855008913308160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076529.650525:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913308160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076529.650538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076529.650544:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076529.650550:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076529.650585:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076529.650619:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076529.650653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076529/real 1769076529] req@ffff9350100caa00 x1855008913308288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076529.650671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913308288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076529.650684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076529.650689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076529.650694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076529.841166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076530.353205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076530.353217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076530.353219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076530.353323:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076530.673212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076530.865167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076531.377217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076531.377243:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076531.377251:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076531.377302:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076531.697200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076531.889191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076532.401223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076532.401224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076532.401224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076532.401274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076532.721128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076532.913169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076533.425121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076533.425149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076533.425178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076533.425190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076533.745176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076533.937104:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076534.449150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076534.449223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076534.449254:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076534.449259:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076534.769145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076534.769231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076534.769240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076534.769245:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076534.769256:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2240 00000100:00080000:3.0:1769076534.769270:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076534.769279:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076534.769324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913308416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076534.769374:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076534.769379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076534.769382:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076534.769387:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2240 00000100:00080000:3.0:1769076534.769392:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076534.769397:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076534.769404:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913308416:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076534.769409:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076534.769429:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913308544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076534.769442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076534.769447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076534.769449:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076534.769454:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2240 00000100:00080000:3.0:1769076534.769459:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076534.769464:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076534.769471:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076534.769480:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076534.769488:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913308672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076534.769501:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076534.769505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076534.769507:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076534.769512:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2240 00000100:00080000:3.0:1769076534.769518:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076534.769523:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076534.769526:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076534/real 1769076534] req@ffff935002a7c380 x1855008913308416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076534.769532:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076534.769548:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913308416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076534.769555:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913308800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076534.769559:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076534.769568:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076534.769574:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913308416:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076534.769587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076534.769592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076534.769594:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076534.769600:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2240 00000100:00080000:3.0:1769076534.769605:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076534.769607:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913308544:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076534.769614:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076534.769622:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076534.769631:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076534.769642:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913308928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076534.769646:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076534/real 1769076534] req@ffff935002a7f100 x1855008913308544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076534.769675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913308544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076534.769676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076534.769687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076534.769692:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076534.769699:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2240 00000100:00080000:3.0:1769076534.769708:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076534.769710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076534.769717:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076534.769717:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076534.769732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913308544:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769076534.769737:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076534.769740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913308672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076534.769760:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913309056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076534.769764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076534.769775:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076534/real 1769076534] req@ffff935002a7e300 x1855008913308672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076534.769815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913308672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076534.769818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076534.769829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076534.769831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076534.769833:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076534.769838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076534.769844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913308672:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076534.769844:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2240 00000100:00100000:0.0:1769076534.769855:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913308800:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076534.769855:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076534.769866:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076534.769874:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076534.769875:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076534.769886:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076534/real 1769076534] req@ffff935002a7ca80 x1855008913308800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076534.769896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913309184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076534.769905:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913308800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076534.769930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076534.769934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076534.769938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913308800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076534.769957:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308928:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076534.769973:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076534.769981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076534/real 1769076534] req@ffff935002a7c380 x1855008913308928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076534.769990:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913308928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076534.769996:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076534.769998:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076534.770001:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913308928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076534.770004:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076534.770014:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076534.770020:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076534/real 1769076534] req@ffff935002a7fb80 x1855008913309056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076534.770028:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913309056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076534.770034:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076534.770036:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076534.770037:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076534.770041:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076534.770070:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076534.770079:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076534/real 1769076534] req@ffff935002a7c700 x1855008913309184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076534.770087:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913309184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076534.770093:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076534.770096:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076534.770098:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076534.961141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076535.473157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076535.473158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076535.473159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076535.473191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076535.793167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076535.985119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076536.497145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076536.497148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076536.497149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076536.497189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076536.817182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076537.009161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076537.521173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076537.521185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076537.521219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076537.521288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076537.841126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076538.033152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076538.545132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076538.545140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076538.545141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076538.545201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076538.865146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076539.057151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076539.569109:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076539.569121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076539.569153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076539.569207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076539.889131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076539.889145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.889152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076539.889157:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.889166:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2245 00000100:00080000:1.0:1769076539.889178:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.889185:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076539.889235:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913309312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076539.889265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.889271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076539.889274:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.889279:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2245 00000100:00080000:1.0:1769076539.889285:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076539.889293:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913309312:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076539.889294:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076539.889311:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076539.889330:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913309440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076539.889340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.889343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076539.889344:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.889347:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2245 00000100:00080000:1.0:1769076539.889358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.889361:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076539.889366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076539.889372:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076539.889377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913309568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076539.889385:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.889387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076539.889389:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.889392:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2245 00000100:00080000:1.0:1769076539.889395:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.889397:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076539.889401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076539.889410:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913309696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076539.889416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076539.889418:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff9350100cb800 x1855008913309312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076539.889421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076539.889422:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.889426:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2245 00000100:00080000:1.0:1769076539.889428:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.889432:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076539.889436:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076539.889450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913309824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076539.889456:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913309312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076539.889458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.889467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076539.889470:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.889478:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2245 00000100:00080000:1.0:1769076539.889483:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.889488:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076539.889490:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769076539.889492:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076539.889496:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076539.889501:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913309312:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076539.889503:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913309952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076539.889510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.889513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076539.889515:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.889517:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2245 00000100:00080000:1.0:1769076539.889519:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.889522:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076539.889526:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913309440:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769076539.889527:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076539.889542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913310080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076539.889553:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076539.889566:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff9350100caa00 x1855008913309440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.889575:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913309440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.889581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.889584:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076539.889586:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913309440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076539.889590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913309568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076539.889601:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076539.889607:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff9350100c9500 x1855008913309568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.889615:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913309568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.889621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.889622:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076539.889624:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913309568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076539.889628:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913309696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076539.889638:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076539.889643:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff9350100c9880 x1855008913309696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.889651:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913309696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.889657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.889658:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076539.889660:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913309696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076539.889663:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076539.889672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076539.889678:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff9350100c8e00 x1855008913309824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.889685:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913309824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.889691:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.889693:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076539.889695:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076539.889698:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076539.889707:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076539.889713:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff9350100ca680 x1855008913309952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.889721:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913309952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.889726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.889728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076539.889729:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913309952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076539.889741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076539.889751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076539.889757:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff9350100ca300 x1855008913310080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.889764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913310080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.889770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.889771:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076539.889773:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076540.081122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076540.593121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076540.593131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076540.593161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076540.593170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076540.913165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076541.105158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076541.617159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076541.617210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076541.617266:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076541.617266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076541.937131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076542.129165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076542.641204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076542.641206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076542.641211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076542.641211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076542.961143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076543.153149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076543.665155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076543.665159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076543.665165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076543.665289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076543.985128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076544.177112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076544.689113:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076544.689118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076544.689188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076544.689210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076545.009171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076545.009181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076545.009186:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076545.009195:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2250 00000100:00080000:1.0:1769076545.009207:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076545.009215:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076545.009221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076545.009253:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913310208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076545.009284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076545.009288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076545.009290:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076545.009296:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2250 00000100:00080000:1.0:1769076545.009298:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076545.009301:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076545.009333:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076545.009346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913310336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076545.009356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913310208:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076545.009365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076545.009369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076545.009370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076545.009374:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2250 00000100:00080000:1.0:1769076545.009377:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076545.009380:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076545.009400:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076545.009409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913310464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076545.009418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076545.009422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076545.009424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076545.009428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2250 00000100:00080000:1.0:1769076545.009432:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076545.009436:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076545.009441:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769076545.009448:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076545.009455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913310592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076545.009468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076545.009472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076545.009474:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076545.009477:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2250 00000100:00080000:1.0:1769076545.009482:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076545.009486:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076545.009490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076545.009495:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076545/real 1769076545] req@ffff9350100c8380 x1855008913310208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076545.009503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913310720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076545.009516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076545.009519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076545.009520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076545.009523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2250 00000100:00080000:1.0:1769076545.009526:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076545.009529:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913310208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076545.009531:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076545.009538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076545.009550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913310848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076545.009550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076545.009562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076545.009562:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076545.009568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076545.009571:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076545.009571:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913310208:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076545.009580:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2250 00000100:00080000:1.0:1769076545.009584:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076545.009589:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076545.009594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076545.009605:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913310336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076545.009608:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913310976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076545.009655:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076545.009674:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076545/real 1769076545] req@ffff9350100c9f80 x1855008913310336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076545.009689:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913310336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076545.009700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076545.009704:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076545.009708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913310336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076545.009716:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913310464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076545.009737:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076545.009765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076545/real 1769076545] req@ffff9350100cad80 x1855008913310464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076545.009780:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913310464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076545.009790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076545.009794:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076545.009798:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913310464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076545.009804:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913310592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076545.009824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076545.009836:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076545/real 1769076545] req@ffff9350100c8000 x1855008913310592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076545.009856:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913310592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076545.009865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076545.009869:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076545.009873:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913310592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076545.009879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310720:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076545.009898:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076545.009907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076545/real 1769076545] req@ffff9350100cb100 x1855008913310720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076545.009915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913310720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076545.009920:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076545.009923:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076545.009925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076545.009929:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076545.009939:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076545.009944:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076545/real 1769076545] req@ffff9350100c9180 x1855008913310848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076545.009952:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913310848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076545.009961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076545.009964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076545.009966:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076545.009979:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076545.009989:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076545.009994:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076545/real 1769076545] req@ffff9350100c9c00 x1855008913310976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076545.010002:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913310976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076545.010007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076545.010009:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076545.010011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913310976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076545.201112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076545.713163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076545.713176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076545.713176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076545.713233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076546.033144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076546.225149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076546.737164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076546.737165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076546.737167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076546.737206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076547.057160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076547.249142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076547.761137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076547.761146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076547.761157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076547.761169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076548.081137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076548.273157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076548.785127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076548.785128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076548.785147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076548.785159:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076549.105131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076549.297129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076549.809129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076549.809138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076549.809179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076549.809274:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076550.129126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076550.129163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076550.129170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076550.129174:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076550.129181:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2255 00000100:00080000:3.0:1769076550.129191:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076550.129197:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076550.129234:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913311104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076550.129257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076550.129260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076550.129261:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076550.129264:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2255 00000100:00080000:3.0:1769076550.129266:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076550.129269:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076550.129276:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076550.129286:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913311232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076550.129294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076550.129296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076550.129297:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076550.129300:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2255 00000100:00080000:3.0:1769076550.129302:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076550.129304:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076550.129308:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076550.129312:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913311104:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076550.129318:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913311360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076550.129327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076550.129329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076550.129330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076550.129333:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2255 00000100:00080000:3.0:1769076550.129337:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076550.129339:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076550.129342:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076550.129349:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913311488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076550.129356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076550.129359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076550.129360:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076550.129363:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2255 00000100:00080000:3.0:1769076550.129383:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076550.129386:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076550.129389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076550.129396:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076550.129399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913311616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076550.129407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076550.129410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076550.129411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076550.129413:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2255 00000100:00080000:3.0:1769076550.129415:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076550.129418:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076550.129420:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076550.129428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913311744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076550.129434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076550.129437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076550.129438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076550.129441:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2255 00000100:00080000:3.0:1769076550.129444:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076550.129446:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076550.129449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076550.129452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076550/real 1769076550] req@ffff935002a7dc00 x1855008913311104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076550.129459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913311872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076550.129492:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913311104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076550.129506:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076550.129511:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076550.129516:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913311104:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076550.129525:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913311232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076550.129549:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076550.129564:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076550/real 1769076550] req@ffff935002a7d180 x1855008913311232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076550.129579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913311232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076550.129589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076550.129592:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076550.129596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913311232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076550.129621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913311360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076550.129642:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076550.129655:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076550/real 1769076550] req@ffff935002a7df80 x1855008913311360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076550.129696:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913311360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076550.129707:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076550.129710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076550.129714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913311360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076550.129720:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913311488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076550.129739:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076550.129750:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076550/real 1769076550] req@ffff935002a7c000 x1855008913311488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076550.129763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913311488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076550.129772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076550.129774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076550.129777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913311488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076550.129781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913311616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076550.129795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076550.129802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076550/real 1769076550] req@ffff935002a7f800 x1855008913311616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076550.129833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913311616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076550.129840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076550.129842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076550.129844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913311616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076550.129848:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913311744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076550.129862:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076550.129869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076550/real 1769076550] req@ffff935002a7c700 x1855008913311744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076550.129876:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913311744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076550.129886:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076550.129888:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076550.129890:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913311744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076550.129894:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913311872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076550.129903:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076550.129908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076550/real 1769076550] req@ffff935002a7fb80 x1855008913311872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076550.129915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913311872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076550.129921:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076550.129922:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076550.129924:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913311872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076550.321121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076550.833113:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076550.833124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076550.833135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076550.833199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076551.153131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076551.345105:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076551.857110:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076551.857111:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076551.857152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076551.857182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076552.177141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076552.369183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076552.881153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076552.881154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076552.881154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076552.881186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076553.201151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076553.393111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076553.905139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076553.905139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076553.905147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076553.905265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076554.225160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076554.417141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076554.929130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076554.929133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076554.929138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076554.929296:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076555.249120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076555.249128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076555.249132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076555.249139:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076555.249149:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2260 00000100:00080000:3.0:1769076555.249169:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076555.249178:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076555.249217:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913312000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076555.249250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076555.249254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076555.249257:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076555.249261:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2260 00000100:00080000:3.0:1769076555.249265:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076555.249270:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076555.249281:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076555.249294:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913312000:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076555.249296:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913312128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076555.249312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076555.249315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076555.249318:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076555.249322:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2260 00000100:00080000:3.0:1769076555.249327:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076555.249331:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076555.249337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076555.249348:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913312256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076555.249359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076555.249363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769076555.249367:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076555.249367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076555.249375:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2260 00000100:00080000:3.0:1769076555.249380:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076555.249385:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076555.249390:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076555.249402:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913312384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076555.249410:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076555/real 1769076555] req@ffff935002a7c380 x1855008913312000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076555.249418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076555.249424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076555.249426:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076555.249433:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913312000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076555.249434:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2260 00000100:00080000:3.0:1769076555.249441:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076555.249449:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076555.249450:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076555.249458:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769076555.249458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076555.249465:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913312000:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076555.249477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913312512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076555.249488:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913312128:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076555.249489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076555.249496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076555.249499:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076555.249504:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2260 00000100:00080000:3.0:1769076555.249510:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076555.249513:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076555.249518:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076555.249525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076555.249527:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076555/real 1769076555] req@ffff935002a7ca80 x1855008913312128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076555.249541:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913312128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076555.249541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913312640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076555.249559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076555.249567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076555.249571:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076555.249587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076555.249593:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2260 00000100:00080000:2.0:1769076555.249598:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076555.249600:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076555.249603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913312128:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076555.249608:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076555.249612:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913312256:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769076555.249617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076555.249634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913312768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076555.249639:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076555.249650:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076555/real 1769076555] req@ffff935002a7e300 x1855008913312256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076555.249671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913312256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076555.249678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076555.249682:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076555.249685:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913312256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076555.249689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913312384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076555.249699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076555.249706:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076555/real 1769076555] req@ffff935002a7f100 x1855008913312384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076555.249714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913312384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076555.249719:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076555.249721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076555.249723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913312384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076555.249736:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913312512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076555.249746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076555.249753:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076555/real 1769076555] req@ffff935002a7fb80 x1855008913312512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076555.249761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913312512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076555.249766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076555.249769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076555.249771:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913312512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076555.249774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913312640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076555.249783:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076555.249788:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076555/real 1769076555] req@ffff935002a7c700 x1855008913312640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076555.249796:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913312640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076555.249802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076555.249804:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076555.249806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913312640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076555.249809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913312768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076555.249826:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076555.249833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076555/real 1769076555] req@ffff935002a7f800 x1855008913312768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076555.249840:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913312768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076555.249845:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076555.249847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076555.249849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913312768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076555.441129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076555.953134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076555.953134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076555.953135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076555.953205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076556.273135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076556.465123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076556.977165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076556.977179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076556.977184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076556.977205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076557.297172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076557.489120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076558.001140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076558.001140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076558.001149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076558.001256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076558.321237:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076558.513126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076559.025195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076559.025199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076559.025352:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076559.025372:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076559.345150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076559.537163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076560.049137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076560.049141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076560.049173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076560.049258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076560.369156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076560.369156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076560.369168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076560.369174:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076560.369184:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2266 00000100:00080000:3.0:1769076560.369197:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076560.369206:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076560.369249:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913312896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076560.369273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076560.369277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076560.369280:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076560.369285:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2266 00000100:00080000:3.0:1769076560.369290:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076560.369295:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076560.369305:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076560.369307:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913312896:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076560.369319:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913313024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076560.369330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076560.369333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076560.369335:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076560.369339:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2266 00000100:00080000:3.0:1769076560.369342:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076560.369346:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076560.369350:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076560.369359:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913313152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076560.369378:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076560.369378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076560.369381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076560.369383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076560.369387:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2266 00000100:00080000:3.0:1769076560.369390:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076560.369394:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076560.369398:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076560.369410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913313280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076560.369419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076560.369421:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076560/real 1769076560] req@ffff935002a7c000 x1855008913312896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076560.369423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076560.369425:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076560.369430:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2266 00000100:00080000:3.0:1769076560.369433:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076560.369437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076560.369439:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913312896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076560.369442:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076560.369451:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076560.369452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913313408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076560.369456:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076560.369461:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913312896:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076560.369461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076560.369465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076560.369467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076560.369471:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2266 00000100:00080000:3.0:1769076560.369474:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076560.369477:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076560.369482:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076560.369484:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076560.369508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913313536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076560.369518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076560.369521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076560.369524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076560.369527:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2266 00000100:00080000:3.0:1769076560.369531:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076560.369534:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076560.369539:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076560.369539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076560.369550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913313664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076560.369553:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076560/real 1769076560] req@ffff935002a7df80 x1855008913313024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076560.369567:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913313024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076560.369586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076560.369590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076560.369594:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076560.369601:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076560.369623:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076560.369632:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076560/real 1769076560] req@ffff935002a7d180 x1855008913313152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076560.369641:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913313152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076560.369648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076560.369650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076560.369652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076560.369657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076560.369668:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076560.369674:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076560/real 1769076560] req@ffff935002a7dc00 x1855008913313280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076560.369682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913313280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076560.369688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076560.369689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076560.369691:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076560.369695:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913313408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076560.369705:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076560.369710:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076560/real 1769076560] req@ffff935002a7f800 x1855008913313408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076560.369718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913313408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076560.369723:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076560.369725:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076560.369726:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913313408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076560.369741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913313536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076560.369751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076560.369757:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076560/real 1769076560] req@ffff935002a7c700 x1855008913313536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076560.369765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913313536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076560.369774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076560.369776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076560.369778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913313536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076560.369781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913313664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076560.369790:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076560.369795:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076560/real 1769076560] req@ffff935002a7fb80 x1855008913313664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076560.369802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913313664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076560.369807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076560.369809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076560.369811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913313664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076560.561134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076561.073167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076561.073168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076561.073184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076561.073214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076561.393175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076561.585150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076562.097129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076562.097177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076562.097241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076562.097293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076562.417102:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076562.609127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076563.121160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076563.121160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076563.121282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076563.121300:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076563.441166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076563.633134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076564.145142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076564.145155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076564.145185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076564.145262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076564.465162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076564.657121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076565.169150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076565.169150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076565.169150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076565.169225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076565.489132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076565.489155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076565.489162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076565.489166:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076565.489174:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2271 00000100:00080000:1.0:1769076565.489185:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076565.489191:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076565.489228:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913313792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076565.489257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076565.489261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076565.489263:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076565.489268:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2271 00000100:00080000:1.0:1769076565.489272:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076565.489276:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076565.489285:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076565.489299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913313920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076565.489308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076565.489311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076565.489312:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076565.489315:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2271 00000100:00080000:1.0:1769076565.489319:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076565.489323:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076565.489326:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076565.489337:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913314048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076565.489347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076565.489353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076565.489355:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913313792:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076565.489356:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076565.489372:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2271 00000100:00080000:1.0:1769076565.489375:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076565.489378:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076565.489382:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076565.489394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913314176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076565.489416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076565.489419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076565.489421:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076565.489425:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2271 00000100:00080000:1.0:1769076565.489430:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076565.489434:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076565.489439:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076565.489448:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913314304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076565.489463:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076565.489472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076565.489477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076565.489479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076565.489483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2271 00000100:00080000:1.0:1769076565.489486:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076565.489489:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076565.489494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076565.489508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913314432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076565.489518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076565.489522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076565.489526:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076565/real 1769076565] req@ffff9350100c8a80 x1855008913313792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076565.489528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076565.489547:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2271 00000100:00080000:1.0:1769076565.489553:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076565.489558:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913313792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076565.489563:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076565.489572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076565.489587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076565.489589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913314560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076565.489603:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076565.489611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913313792:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076565.489624:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313920:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076565.489647:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076565.489657:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076565/real 1769076565] req@ffff9350100c9c00 x1855008913313920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076565.489666:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913313920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076565.489673:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076565.489675:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076565.489677:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913313920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076565.489700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076565.489712:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076565.489718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076565/real 1769076565] req@ffff9350100c9180 x1855008913314048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076565.489725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913314048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076565.489731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076565.489734:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076565.489736:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076565.489740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076565.489749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076565.489755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076565/real 1769076565] req@ffff9350100cb100 x1855008913314176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076565.489762:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913314176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076565.489767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076565.489769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076565.489771:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076565.489775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913314304:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076565.489785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076565.489791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076565/real 1769076565] req@ffff9350100c8000 x1855008913314304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076565.489799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913314304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076565.489804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076565.489806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076565.489807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913314304:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076565.489811:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913314432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076565.489820:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076565.489825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076565/real 1769076565] req@ffff9350100cad80 x1855008913314432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076565.489833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913314432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076565.489843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076565.489845:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076565.489847:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913314432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076565.489851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913314560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076565.489860:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076565.489866:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076565/real 1769076565] req@ffff9350100c9f80 x1855008913314560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076565.489872:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913314560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076565.489877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076565.489879:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076565.489880:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913314560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076565.681110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076566.193115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076566.193119:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076566.193129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076566.193180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076566.513139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076566.705117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076567.217125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076567.217130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076567.217130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076567.217177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076567.537117:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076567.729133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076568.241121:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076568.241121:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076568.241152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076568.241169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076568.561131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076568.753115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076569.265180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076569.265181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076569.265181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076569.265290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076569.585149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076569.777186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076570.289172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076570.289180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076570.289237:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076570.289389:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076570.609169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076570.609173:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076570.609186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076570.609192:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076570.609205:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2276 00000100:00080000:1.0:1769076570.609220:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076570.609230:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076570.609281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913314688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076570.609346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076570.609351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076570.609354:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076570.609359:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2276 00000100:00080000:1.0:1769076570.609364:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076570.609369:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076570.609381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076570.609398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913314816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076570.609411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076570.609415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076570.609418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076570.609423:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2276 00000100:00080000:1.0:1769076570.609428:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076570.609435:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913314688:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076570.609436:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076570.609448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076570.609461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913314944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076570.609473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076570.609476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076570.609479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076570.609483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2276 00000100:00080000:1.0:1769076570.609487:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076570.609491:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076570.609496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076570.609508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913315072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076570.609518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076570.609521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076570.609523:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076570.609529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2276 00000800:00000100:2.0:1769076570.609537:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076570.609540:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076570.609547:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076570.609553:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076570.609566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913315200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076570.609577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076570.609581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076570.609583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076570.609589:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2276 00000100:00080000:1.0:1769076570.609595:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076570.609602:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076570/real 1769076570] req@ffff9350100c8380 x1855008913314688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076570.609604:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076570.609616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076570.609636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913315328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076570.609649:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913314688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076570.609652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076570.609662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076570.609667:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076570.609679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2276 00000100:00080000:1.0:1769076570.609686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076570.609691:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076570.609696:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076570.609696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076570.609715:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769076570.609716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913315456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076570.609734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913314688:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076570.609754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314816:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076570.609791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076570.609807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076570/real 1769076570] req@ffff9350100ca300 x1855008913314816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076570.609822:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913314816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076570.609832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076570.609836:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076570.609840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076570.609869:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076570.609888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076570.609898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076570/real 1769076570] req@ffff9350100ca680 x1855008913314944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076570.609910:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913314944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076570.609919:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076570.609923:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076570.609926:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913314944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076570.609932:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076570.609947:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076570.609956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076570/real 1769076570] req@ffff9350100c8e00 x1855008913315072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076570.609969:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913315072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076570.609977:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076570.609980:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076570.609983:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076570.609989:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913315200:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076570.610004:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076570.610013:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076570/real 1769076570] req@ffff9350100c9880 x1855008913315200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076570.610025:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913315200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076570.610034:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076570.610036:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076570.610039:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913315200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076570.610045:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913315328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076570.610107:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076570.610120:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076570/real 1769076570] req@ffff9350100c9500 x1855008913315328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076570.610132:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913315328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076570.610149:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076570.610152:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076570.610156:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913315328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076570.610162:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913315456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076570.610178:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076570.610187:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076570/real 1769076570] req@ffff9350100caa00 x1855008913315456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076570.610199:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913315456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076570.610208:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076570.610210:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076570.610214:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913315456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076570.801182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076571.313182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076571.313182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076571.313182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076571.313218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076571.633207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076571.825266:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076572.337122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076572.337213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076572.337218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076572.337232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076572.657124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076572.849110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076573.361180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076573.361195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076573.361198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076573.361361:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076573.681145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076573.873209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076574.385164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076574.385173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076574.385187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076574.385293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076574.705291:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076574.897204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076575.409233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076575.409250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076575.409291:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076575.409399:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076575.729201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076575.729223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076575.729270:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076575.729276:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076575.729286:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2281 00000100:00080000:2.0:1769076575.729300:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076575.729307:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076575.729353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913315584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076575.729391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076575.729395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076575.729397:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076575.729402:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2281 00000100:00080000:2.0:1769076575.729405:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076575.729408:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076575.729418:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076575.729433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913315712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076575.729444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076575.729448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076575.729449:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076575.729460:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2281 00000100:00100000:0.0:1769076575.729461:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913315584:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076575.729476:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076575.729483:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076575.729489:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076575.729524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913315840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076575.729535:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076575.729538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076575.729541:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076575.729547:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2281 00000100:00080000:2.0:1769076575.729554:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076575.729557:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076575.729563:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076575.729577:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913315968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076575.729588:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076575.729602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076575.729607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076575.729609:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076575.729618:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2281 00000100:00080000:2.0:1769076575.729624:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076575.729628:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076575.729633:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076575.729646:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913316096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076575.729657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076575.729661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076575.729664:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076575.729669:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2281 00000100:00080000:2.0:1769076575.729673:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076575.729683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076575/real 1769076575] req@ffff935002a63100 x1855008913315584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076575.729684:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076575.729694:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076575.729711:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913316224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076575.729725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076575.729727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913315584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076575.729731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076575.729739:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076575.729760:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2281 00000100:00080000:0.0:1769076575.729773:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076575.729782:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076575.729789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076575.729798:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076575.729805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913315584:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076575.729808:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076575.729835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913316352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076575.729840:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315712:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076575.729897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076575.729927:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076575/real 1769076575] req@ffff935002a61f80 x1855008913315712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076575.729944:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913315712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076575.729953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076575.729956:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076575.729959:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076575.729979:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076575.730008:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076575.730019:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076575/real 1769076575] req@ffff935002a61c00 x1855008913315840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076575.730035:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913315840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076575.730043:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076575.730046:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076575.730076:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076575.730082:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076575.730106:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076575.730125:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076575/real 1769076575] req@ffff935002a61180 x1855008913315968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076575.730142:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913315968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076575.730154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076575.730159:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076575.730164:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913315968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076575.730173:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316096:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076575.730197:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076575.730208:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076575/real 1769076575] req@ffff935002a61500 x1855008913316096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076575.730219:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913316096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076575.730254:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076575.730258:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076575.730261:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076575.730267:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076575.730288:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076575.730298:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076575/real 1769076575] req@ffff935002a62300 x1855008913316224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076575.730307:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913316224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076575.730315:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076575.730318:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076575.730320:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076575.730342:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076575.730356:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076575.730364:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076575/real 1769076575] req@ffff935002a62680 x1855008913316352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076575.730373:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913316352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076575.730380:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076575.730382:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076575.730385:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076575.921214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076576.433167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076576.433193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076576.433199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076576.433273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076576.753125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076576.945162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076577.457111:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076577.457127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076577.457129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076577.457181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076577.777128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076577.969146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076578.481177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076578.481177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076578.481283:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076578.481311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076578.801165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076578.993229:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076579.505204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076579.505205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076579.505288:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076579.505412:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076579.825154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076580.017209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076580.529173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076580.529173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076580.529198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076580.529228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076580.849108:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076580.849137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076580.849160:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076580.849166:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076580.849188:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2286 00000100:00080000:2.0:1769076580.849200:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076580.849207:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076580.849249:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913316480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076580.849280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076580.849284:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076580.849286:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076580.849289:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2286 00000100:00080000:2.0:1769076580.849292:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076580.849296:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076580.849306:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076580.849328:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913316608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076580.849339:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913316480:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076580.849339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076580.849349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076580.849351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076580.849355:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2286 00000100:00080000:2.0:1769076580.849357:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076580.849360:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076580.849365:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076580.849375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913316736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076580.849391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076580.849394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076580.849395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076580.849398:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2286 00000100:00080000:2.0:1769076580.849401:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076580.849404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076580.849408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076580.849416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913316864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076580.849422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076580.849426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076580.849428:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076580.849431:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2286 00000100:00080000:2.0:1769076580.849434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076580.849436:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076580.849440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076580.849441:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076580.849462:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913316992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076580.849471:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076580.849474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076580.849476:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076580.849479:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2286 00000100:00080000:2.0:1769076580.849481:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076580.849484:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076580.849487:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076580.849497:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913317120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076580.849502:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff935002a60a80 x1855008913316480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076580.849508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076580.849526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076580.849528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076580.849534:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2286 00000100:00100000:0.0:1769076580.849537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913316480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076580.849539:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076580.849547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076580.849555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076580.849558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076580.849567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913317248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076580.849568:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076580.849577:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913316480:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076580.849608:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913316608:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076580.849627:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076580.849637:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff935002a60000 x1855008913316608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076580.849646:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913316608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076580.849654:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076580.849656:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076580.849659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913316608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076580.849663:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913316736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076580.849675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076580.849682:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff935002a62a00 x1855008913316736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076580.849690:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913316736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076580.849696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076580.849698:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076580.849700:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913316736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076580.849703:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913316864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076580.849714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076580.849720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff935002a61880 x1855008913316864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076580.849728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913316864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076580.849737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076580.849741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076580.849746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913316864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076580.849753:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316992:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076580.849775:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076580.849789:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff935002a62680 x1855008913316992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076580.849811:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913316992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076580.849822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076580.849825:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076580.849828:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913316992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076580.849833:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913317120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076580.849850:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076580.849863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff935002a62300 x1855008913317120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076580.849877:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913317120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076580.849886:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076580.849890:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076580.849894:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913317120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076580.849902:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913317248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076580.849924:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076580.849935:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff935002a61500 x1855008913317248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076580.849944:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913317248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076580.849960:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076580.849962:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076580.849964:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913317248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076581.041175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076581.553163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076581.553163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076581.553233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076581.553364:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076581.873142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076582.065227:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076582.577185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076582.577186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076582.577186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076582.577246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076582.897193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076583.089169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076583.601157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076583.601163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076583.601233:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076583.601337:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076583.921276:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076584.113251:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076584.625154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076584.625155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076584.625162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076584.625239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076584.945179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076585.137184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076585.649183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076585.649224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076585.649226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076585.649284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076585.969254:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076585.969267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076585.969283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076585.969291:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076585.969308:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2291 00000100:00080000:3.0:1769076585.969328:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076585.969342:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076585.969429:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913317376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076585.969491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076585.969495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076585.969498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076585.969503:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2291 00000100:00080000:3.0:1769076585.969507:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076585.969512:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076585.969544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076585.969566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913317504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076585.969583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076585.969588:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076585.969590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076585.969600:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2291 00000100:00100000:0.0:1769076585.969604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913317376:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076585.969609:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076585.969623:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076585.969631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076585.969652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913317632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076585.969663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076585.969667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076585.969670:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076585.969676:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2291 00000100:00080000:3.0:1769076585.969683:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076585.969688:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076585.969693:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076585.969707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913317760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076585.969735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076585.969739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076585.969741:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076585.969744:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2291 00000100:00080000:3.0:1769076585.969749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076585.969754:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076585.969761:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076585.969787:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913317888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076585.969802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076585.969809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913317504:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076585.969811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076585.969829:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076585.969843:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2291 00000100:00080000:3.0:1769076585.969856:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076585.969863:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076585.969873:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076585.969889:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913317632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076585.969919:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913318016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076585.969939:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913317760:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076585.969943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076585.969953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076585.969957:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076585.969969:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2291 00000100:00080000:3.0:1769076585.969980:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076585.969987:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076585.969995:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913317888:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769076585.970009:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076585.970043:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076585.970144:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913318144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076585.970204:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076586.161190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076586.673236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076586.673344:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076586.673661:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076586.673697:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076586.993357:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076587.185123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076587.697170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076587.697208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076587.697218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076587.697227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076588.017156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076588.209130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076588.721114:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076588.721114:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076588.721138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076588.721262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076589.041157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:1.0:1769076589.041473:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769076589.041523:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076589.041549:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076589.041607:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076589.041619:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076589.041657:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076589.041673:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076589] req@ffff935002a7e300 x1855008913317504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1769076589.041674:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076589.041692:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769076589.041718:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076589.041722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913317504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076589.041743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076589.041751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076589.041757:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913317504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076589.041777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076589] req@ffff935002a7ca80 x1855008913317632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076589.041796:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913317632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076589.041806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076589.041810:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076589.041815:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913317632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076589.041824:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076589] req@ffff935002a7c380 x1855008913317760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076589.041847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913317760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076589.041857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076589.041861:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076589.041866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913317760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076589.041875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076589] req@ffff935002a7f100 x1855008913317376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076589.041902:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913317376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076589.041913:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076589.041918:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076589.041923:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913317376:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076589.041962:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076589] req@ffff935002a7f800 x1855008913318144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076589.041980:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913318144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076589.041991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076589.041996:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076589.042001:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076589.042026:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076589] req@ffff935002a7fb80 x1855008913317888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076589.042044:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913317888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076589.042090:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076589.042097:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076589.042103:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913317888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076589.042114:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076589] req@ffff935002a7c700 x1855008913318016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076589.042137:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913318016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076589.042147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076589.042152:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076589.042157:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076589.233173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076589.745190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076589.745198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076589.745213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076589.745275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076590.065201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076590.257186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076590.769176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076590.769177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076590.769183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076590.769226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076591.089129:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076591.089131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076591.089142:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076591.089146:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076591.089155:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2296 00000100:00080000:3.0:1769076591.089167:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076591.089175:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076591.089210:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913318272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076591.089234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076591.089237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076591.089238:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076591.089241:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2296 00000100:00080000:3.0:1769076591.089244:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076591.089246:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076591.089255:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076591.089275:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913318400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076591.089284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076591.089286:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913318272:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076591.089287:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076591.089291:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076591.089298:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2296 00000100:00080000:3.0:1769076591.089301:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076591.089304:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076591.089307:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076591.089317:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913318528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076591.089325:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076591.089328:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076591.089330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076591.089335:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2296 00000100:00080000:3.0:1769076591.089339:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076591.089344:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076591.089349:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076591.089362:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913318656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076591.089371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076591.089377:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076591.089378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076591.089383:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076591.089389:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2296 00000100:00080000:3.0:1769076591.089395:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076591.089401:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076591.089408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076591.089421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913318784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076591.089428:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076591/real 1769076591] req@ffff935002a7dc00 x1855008913318272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076591.089446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076591.089459:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913318272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076591.089478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076591.089480:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076591.089482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076591.089490:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076591.089491:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2296 00000100:00080000:3.0:1769076591.089500:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076591.089504:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076591.089510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076591.089523:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913318272:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076591.089529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913318912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076591.089544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076591.089548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076591.089550:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076591.089557:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913318400:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076591.089558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2296 00000100:00080000:3.0:1769076591.089565:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076591.089571:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076591.089577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076591.089592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913319040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076591.089596:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076591.089617:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076591/real 1769076591] req@ffff935002a7d180 x1855008913318400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076591.089634:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913318400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076591.089646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076591.089651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076591.089654:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913318400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076591.089662:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913318528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076591.089684:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076591.089697:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076591/real 1769076591] req@ffff935002a7df80 x1855008913318528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076591.089709:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913318528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076591.089717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076591.089721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076591.089724:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913318528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076591.089731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913318656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076591.089751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076591.089761:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076591/real 1769076591] req@ffff935002a7c000 x1855008913318656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076591.089773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913318656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076591.089782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076591.089786:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076591.089790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913318656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076591.089796:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318784:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076591.089815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076591.089825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076591/real 1769076591] req@ffff935002a7c700 x1855008913318784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076591.089836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913318784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076591.089845:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076591.089848:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076591.089852:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076591.089858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076591.089876:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076591.089887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076591/real 1769076591] req@ffff935002a7fb80 x1855008913318912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076591.089901:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913318912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076591.089909:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076591.089913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076591.089916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913318912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076591.089940:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076591.089959:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076591.089969:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076591/real 1769076591] req@ffff935002a7f800 x1855008913319040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076591.089997:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913319040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076591.090005:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076591.090008:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076591.090011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076591.281150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076591.793162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076591.793171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076591.793178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076591.793197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076592.113139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076592.305147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076592.817118:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076592.817119:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076592.817140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076592.817173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076593.137161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076593.329146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076593.841157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076593.841157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076593.841158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076593.841191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076594.161141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076594.353106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076594.865134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076594.865134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076594.865158:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076594.865243:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076595.185144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076595.377173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076595.889193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076595.889238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076595.889265:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076595.889606:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076596.209164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076596.209194:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076596.209200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076596.209205:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076596.209215:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2301 00000100:00080000:1.0:1769076596.209228:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076596.209234:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076596.209284:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913319168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076596.209307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076596.209310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076596.209312:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076596.209315:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2301 00000100:00080000:1.0:1769076596.209318:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076596.209321:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076596.209332:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076596.209340:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913319168:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076596.209346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913319296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076596.209355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076596.209357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076596.209358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076596.209361:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2301 00000100:00080000:1.0:1769076596.209364:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076596.209366:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076596.209369:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076596.209378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913319424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076596.209385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076596.209389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076596.209390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076596.209393:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2301 00000100:00080000:1.0:1769076596.209410:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076596.209412:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076596.209416:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076596.209424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913319552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076596.209427:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076596.209436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076596.209439:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076596.209441:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076596.209445:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2301 00000100:00080000:1.0:1769076596.209448:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076596.209451:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076596.209454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076596.209462:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913319680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076596.209469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076596.209472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076596.209476:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076596/real 1769076596] req@ffff9350100cb800 x1855008913319168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076596.209476:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076596.209482:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2301 00000100:00080000:1.0:1769076596.209489:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076596.209495:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076596.209499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076596.209503:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913319168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076596.209511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913319808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076596.209531:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076596.209539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076596.209552:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076596.209554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076596.209557:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913319168:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076596.209557:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076596.209564:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2301 00000100:00080000:1.0:1769076596.209567:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076596.209570:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076596.209573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076596.209580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913319936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076596.209587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913319296:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076596.209607:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076596.209616:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076596/real 1769076596] req@ffff9350100caa00 x1855008913319296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076596.209624:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913319296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076596.209631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076596.209633:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076596.209635:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913319296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076596.209639:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913319424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076596.209650:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076596.209659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076596/real 1769076596] req@ffff9350100c9500 x1855008913319424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076596.209671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913319424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076596.209680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076596.209684:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076596.209687:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913319424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076596.209693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913319552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076596.209713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076596.209724:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076596/real 1769076596] req@ffff9350100c9880 x1855008913319552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076596.209732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913319552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076596.209737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076596.209739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076596.209741:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913319552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076596.209745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319680:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076596.209767:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076596.209790:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076596/real 1769076596] req@ffff9350100c8e00 x1855008913319680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076596.209798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913319680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076596.209808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076596.209810:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076596.209812:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076596.209815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076596.209833:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076596.209842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076596/real 1769076596] req@ffff9350100ca680 x1855008913319808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076596.209854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913319808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076596.209862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076596.209866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076596.209870:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076596.209907:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076596.209924:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076596.209932:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076596/real 1769076596] req@ffff9350100ca300 x1855008913319936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076596.209941:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913319936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076596.209946:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076596.209960:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076596.209963:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913319936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076596.401151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076596.913216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076596.913229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076596.913270:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076596.913355:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076597.233187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076597.425123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076597.937198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076597.937209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076597.937210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076597.937254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076598.257145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076598.449166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076598.961154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076598.961163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076598.961207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076598.961223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076599.281171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076599.473139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076599.985157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076599.985162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076599.985163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076599.985203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076600.305150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076600.497186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076601.009179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076601.009180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076601.009268:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076601.009346:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076601.329189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076601.329221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076601.329230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076601.329236:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076601.329248:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2307 00000100:00080000:2.0:1769076601.329262:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076601.329270:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076601.329321:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913320064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076601.329354:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076601.329358:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076601.329360:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076601.329364:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2307 00000100:00080000:2.0:1769076601.329368:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076601.329374:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076601.329388:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076601.329401:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913320192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076601.329408:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913320064:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076601.329418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076601.329423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076601.329424:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076601.329430:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2307 00000100:00080000:2.0:1769076601.329433:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076601.329437:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076601.329442:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076601.329457:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913320320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076601.329467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076601.329470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076601.329472:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076601.329476:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2307 00000100:00080000:2.0:1769076601.329482:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076601.329486:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076601.329490:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076601.329496:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076601.329503:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913320448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076601.329516:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076601.329520:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076601.329523:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076601.329526:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2307 00000100:00080000:2.0:1769076601.329530:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076601.329534:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076601.329538:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076601.329553:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076601/real 1769076601] req@ffff935002a61180 x1855008913320064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076601.329555:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913320576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076601.329591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076601.329596:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913320064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076601.329600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076601.329607:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076601.329616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076601.329623:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076601.329627:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913320064:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076601.329634:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2307 00000100:00080000:2.0:1769076601.329643:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076601.329651:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076601.329658:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913320192:192.168.204.121@tcp:38:kworker.0 00000008:00100000:2.0:1769076601.329661:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076601.329683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913320704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076601.329694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076601.329708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076601.329713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076601.329718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076601/real 1769076601] req@ffff935002a61c00 x1855008913320192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076601.329718:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076601.329729:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2307 00000100:00080000:2.0:1769076601.329738:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076601.329746:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076601.329751:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913320192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076601.329755:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076601.329772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076601.329775:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913320832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076601.329781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076601.329792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913320192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076601.329838:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913320320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076601.329883:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076601.329896:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076601/real 1769076601] req@ffff935002a61f80 x1855008913320320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076601.329909:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913320320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076601.329918:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076601.329922:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076601.329925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913320320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076601.329931:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913320448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076601.329947:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076601.329955:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076601/real 1769076601] req@ffff935002a63100 x1855008913320448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076601.329966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913320448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076601.329975:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076601.329978:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076601.329982:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913320448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076601.329988:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913320576:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076601.330003:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076601.330012:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076601/real 1769076601] req@ffff935002a61500 x1855008913320576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076601.330023:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913320576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076601.330032:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076601.330035:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076601.330038:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913320576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076601.330057:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913320704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076601.330107:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076601.330119:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076601/real 1769076601] req@ffff935002a62300 x1855008913320704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076601.330132:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913320704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076601.330140:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076601.330144:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076601.330148:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913320704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076601.330154:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913320832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076601.330170:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076601.330180:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076601/real 1769076601] req@ffff935002a62680 x1855008913320832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076601.330192:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913320832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076601.330200:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076601.330203:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076601.330206:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913320832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076601.521165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076602.033208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076602.033209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076602.033218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076602.033328:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076602.353154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076602.545112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076603.057126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076603.057131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076603.057191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076603.057243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076603.377213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076603.569196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076604.081190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076604.081205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076604.081257:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076604.081277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076604.401123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076604.593169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076605.105212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076605.105232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076605.105262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076605.105272:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076605.425183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076605.617146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076606.129254:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076606.129257:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076606.129264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076606.129336:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076606.449168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076606.449176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076606.449181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076606.449189:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076606.449200:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2312 00000100:00080000:0.0:1769076606.449214:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076606.449223:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076606.449297:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913320960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076606.449357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076606.449361:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076606.449363:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076606.449369:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2312 00000100:00080000:0.0:1769076606.449373:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076606.449379:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076606.449391:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076606.449407:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913320960:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076606.449408:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913321088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076606.449444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076606.449449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076606.449453:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076606.449459:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2312 00000100:00080000:0.0:1769076606.449477:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076606.449502:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076606.449510:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769076606.449524:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076606.449533:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913321216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076606.449550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076606.449555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076606.449558:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076606.449566:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2312 00000100:00100000:2.0:1769076606.449589:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076606/real 1769076606] req@ffff9350103f9180 x1855008913320960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076606.449600:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076606.449614:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076606.449622:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076606.449628:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913320960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076606.449641:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913321344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076606.449643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076606.449657:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076606.449665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076606.449668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913320960:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076606.449671:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076606.449677:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076606.449683:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2312 00000100:00080000:0.0:1769076606.449687:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076606.449691:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076606.449695:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076606.449704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076606.449711:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913321472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076606.449750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076606.449755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769076606.449759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076606.449788:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076606.449794:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2312 00000100:00080000:0.0:1769076606.449800:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076606.449808:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076606.449808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076606/real 1769076606] req@ffff9350103fb100 x1855008913321088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076606.449819:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076606.449834:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913321088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076606.449841:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913321600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076606.449852:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076606.449860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076606.449865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076606.449867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076606.449871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076606.449872:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321088:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076606.449880:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2312 00000100:00080000:0.0:1769076606.449887:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076606.449887:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321216:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769076606.449897:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076606.449903:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076606.449917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913321728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076606.449924:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076606.449941:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076606/real 1769076606] req@ffff9350103f8380 x1855008913321216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076606.449955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913321216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076606.449963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076606.449968:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076606.449971:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076606.449976:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076606.449990:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076606.449998:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076606/real 1769076606] req@ffff9350103f8700 x1855008913321344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076606.450008:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913321344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076606.450015:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076606.450018:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076606.450020:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076606.450036:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913321472:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076606.450049:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076606.450057:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076606/real 1769076606] req@ffff9350103faa00 x1855008913321472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076606.450090:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913321472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076606.450098:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076606.450101:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076606.450104:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913321472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076606.450109:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913321600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076606.450125:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076606.450135:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076606/real 1769076606] req@ffff9350103f9180 x1855008913321600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076606.450145:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913321600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076606.450152:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076606.450155:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076606.450158:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913321600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076606.450162:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913321728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076606.450174:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076606.450182:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076606/real 1769076606] req@ffff9350103f9c00 x1855008913321728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076606.450199:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913321728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076606.450206:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076606.450208:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076606.450211:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913321728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076606.641177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076607.153171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076607.153188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076607.153214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076607.153235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076607.473399:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076607.665185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076608.177229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076608.177229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076608.177298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076608.177322:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076608.497162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076608.689180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076609.201162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076609.201163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076609.201192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076609.201254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076609.521227:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076609.713139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076610.225187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076610.225239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076610.225295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076610.225424:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076610.545136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076610.737137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076611.249140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076611.249211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076611.249293:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076611.249298:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076611.569196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076611.569205:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076611.569226:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076611.569245:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076611.569280:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2317 00000100:00080000:3.0:1769076611.569305:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076611.569313:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076611.569374:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913321856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076611.569436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076611.569440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076611.569443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076611.569448:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2317 00000100:00080000:3.0:1769076611.569452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076611.569456:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076611.569468:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076611.569500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913321984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076611.569509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076611.569512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076611.569513:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076611.569516:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2317 00000100:00080000:3.0:1769076611.569519:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076611.569524:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913321856:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076611.569524:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076611.569533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076611.569542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913322112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076611.569549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076611.569552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076611.569554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076611.569557:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2317 00000100:00080000:3.0:1769076611.569559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076611.569562:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076611.569583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076611.569601:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913322240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076611.569610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076611.569614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076611.569615:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076611.569619:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2317 00000100:00080000:3.0:1769076611.569622:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076611.569657:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076611.569664:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076611.569672:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076611.569701:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913322368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076611.569709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076611.569711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076611.569714:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076611.569717:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2317 00000100:00080000:3.0:1769076611.569721:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076611.569725:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076611.569729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076611/real 1769076611] req@ffff935002a7f100 x1855008913321856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076611.569731:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076611.569744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913322496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076611.569752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076611.569755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913321856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076611.569756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076611.569759:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076611.569773:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2317 00000100:00080000:3.0:1769076611.569778:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076611.569784:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076611.569788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076611.569788:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076611.569798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769076611.569802:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913322624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076611.569804:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913321856:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076611.569819:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321984:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076611.569852:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076611.569861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076611/real 1769076611] req@ffff935002a7c380 x1855008913321984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076611.569871:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913321984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076611.569878:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076611.569880:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076611.569883:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913321984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076611.569904:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913322112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076611.569915:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076611.569922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076611/real 1769076611] req@ffff935002a7ca80 x1855008913322112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076611.569931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913322112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076611.569937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076611.569950:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076611.569952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913322112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076611.569956:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913322240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076611.569994:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076611.570003:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076611/real 1769076611] req@ffff935002a7e300 x1855008913322240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076611.570014:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913322240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076611.570027:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076611.570030:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076611.570033:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913322240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076611.570037:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913322368:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076611.570097:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076611.570109:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076611/real 1769076611] req@ffff935002a7f800 x1855008913322368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076611.570126:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913322368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076611.570138:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076611.570142:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076611.570158:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913322368:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076611.570167:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913322496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076611.570194:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076611.570209:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076611/real 1769076611] req@ffff935002a7fb80 x1855008913322496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076611.570230:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913322496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076611.570238:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076611.570252:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076611.570256:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913322496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076611.570261:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913322624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076611.570279:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076611.570288:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076611/real 1769076611] req@ffff935002a7c700 x1855008913322624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076611.570298:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913322624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076611.570305:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076611.570307:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076611.570310:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913322624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076611.761182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076612.273154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076612.273163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076612.273210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076612.273227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076612.593163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076612.785198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076613.297202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076613.297280:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076613.297319:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076613.297325:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076613.617203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076613.809187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076614.321222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076614.321229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076614.321311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076614.321313:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076614.641141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076614.833172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076615.345204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076615.345297:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076615.345302:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076615.345344:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076615.665169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076615.857115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076616.369126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076616.369128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076616.369259:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076616.369337:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076616.689142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076616.689148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076616.689154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076616.689159:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076616.689167:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2322 00000100:00080000:3.0:1769076616.689179:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076616.689185:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076616.689222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913322752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076616.689246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076616.689249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076616.689251:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076616.689254:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2322 00000100:00080000:3.0:1769076616.689257:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076616.689259:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076616.689267:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076616.689276:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913322880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076616.689285:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076616.689288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076616.689289:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076616.689291:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2322 00000100:00080000:3.0:1769076616.689293:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076616.689295:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076616.689299:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076616.689306:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913323008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076616.689314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076616.689317:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076616.689318:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076616.689322:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2322 00000100:00080000:3.0:1769076616.689324:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076616.689330:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076616.689335:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076616.689336:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913322752:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076616.689344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913323136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076616.689364:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076616.689367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076616.689369:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076616.689374:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2322 00000100:00080000:3.0:1769076616.689378:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076616.689381:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076616.689384:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076616.689396:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913323264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076616.689403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076616.689406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076616.689408:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076616.689411:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2322 00000100:00080000:3.0:1769076616.689425:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076616.689428:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076616.689432:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076616.689439:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076616.689446:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913323392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076616.689454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076616.689458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076616.689460:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076616.689463:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2322 00000100:00080000:3.0:1769076616.689467:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076616.689470:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076616.689473:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076616.689480:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913323520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076616.689508:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076616/real 1769076616] req@ffff935002a7c000 x1855008913322752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076616.689532:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913322752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076616.689561:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076616.689568:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076616.689575:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913322752:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076616.689584:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913322880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076616.689612:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076616.689627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076616/real 1769076616] req@ffff935002a7df80 x1855008913322880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076616.689651:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913322880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076616.689661:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076616.689666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076616.689669:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913322880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076616.689700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076616.689724:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076616.689736:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076616/real 1769076616] req@ffff935002a7d180 x1855008913323008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076616.689749:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913323008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076616.689758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076616.689762:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076616.689765:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076616.689772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076616.689801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076616.689814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076616/real 1769076616] req@ffff935002a7dc00 x1855008913323136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076616.689828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913323136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076616.689837:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076616.689841:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076616.689845:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076616.689852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913323264:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076616.689873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076616.689884:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076616/real 1769076616] req@ffff935002a7c700 x1855008913323264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076616.689897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913323264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076616.689906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076616.689910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076616.689914:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913323264:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076616.689920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913323392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076616.689939:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076616.689951:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076616/real 1769076616] req@ffff935002a7fb80 x1855008913323392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076616.689964:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913323392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076616.689973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076616.689977:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076616.689980:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913323392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076616.689987:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913323520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076616.690007:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076616.690019:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076616/real 1769076616] req@ffff935002a7f800 x1855008913323520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076616.690031:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913323520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076616.690040:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076616.690043:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076616.690047:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913323520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076616.881155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076617.393174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076617.393181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076617.393181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076617.393239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076617.713149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076617.905130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076618.417296:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076618.417297:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076618.417355:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076618.417357:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076618.737162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076618.929226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076619.441149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076619.441166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076619.441216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076619.441273:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076619.761144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076619.953124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076620.465146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076620.465198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076620.465234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076620.465240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076620.785159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076620.977164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076621.489194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076621.489200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076621.489246:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076621.489280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076621.809131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076621.809222:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076621.809231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076621.809236:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076621.809249:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2327 00000100:00080000:1.0:1769076621.809262:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076621.809269:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076621.809313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913323648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076621.809343:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076621.809346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076621.809348:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076621.809351:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2327 00000100:00080000:1.0:1769076621.809354:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076621.809357:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076621.809367:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076621.809376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913323776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076621.809390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076621.809394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076621.809395:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076621.809399:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2327 00000100:00080000:1.0:1769076621.809401:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076621.809404:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076621.809408:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076621.809417:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913323904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076621.809426:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076621.809428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076621.809430:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076621.809436:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2327 00000100:00080000:1.0:1769076621.809439:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076621.809442:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076621.809445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076621.809451:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913323648:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076621.809453:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913324032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076621.809489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076621.809491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076621.809492:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076621.809496:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2327 00000100:00080000:1.0:1769076621.809500:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076621.809505:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076621.809511:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076621.809521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913324160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076621.809528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076621.809531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076621.809533:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076621.809547:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2327 00000100:00080000:1.0:1769076621.809549:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076621.809552:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076621.809555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076621.809562:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913324288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076621.809567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076621.809569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076621.809570:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076621.809572:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2327 00000100:00080000:1.0:1769076621.809576:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076621.809579:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076621.809583:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076621.809589:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913324416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076621.809604:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076621.809670:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076621/real 1769076621] req@ffff9350100c8380 x1855008913323648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076621.809694:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913323648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076621.809708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076621.809714:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076621.809719:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913323648:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076621.809727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323776:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076621.809751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076621.809764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076621/real 1769076621] req@ffff9350100c9f80 x1855008913323776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076621.809778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913323776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076621.809789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076621.809793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076621.809796:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076621.809860:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076621.809902:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076621.809913:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076621/real 1769076621] req@ffff9350100cad80 x1855008913323904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076621.809925:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913323904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076621.809934:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076621.809937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076621.809941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913323904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076621.809948:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076621.809962:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076621.809971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076621/real 1769076621] req@ffff9350100c8000 x1855008913324032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076621.809982:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913324032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076621.809990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076621.809993:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076621.809996:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076621.810002:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913324160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076621.810016:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076621.810025:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076621/real 1769076621] req@ffff9350100cb100 x1855008913324160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076621.810036:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913324160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076621.810045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076621.810047:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076621.810050:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913324160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076621.810056:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913324288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076621.810123:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076621.810136:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076621/real 1769076621] req@ffff9350100c9180 x1855008913324288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076621.810150:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913324288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076621.810158:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076621.810161:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076621.810165:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913324288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076621.810171:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913324416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076621.810188:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076621.810197:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076621/real 1769076621] req@ffff9350100c9c00 x1855008913324416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076621.810208:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913324416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076621.810216:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076621.810220:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076621.810223:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913324416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076622.001155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076622.513143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076622.513154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076622.513186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076622.513244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076622.833207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076623.025150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076623.537157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076623.537160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076623.537193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076623.537223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076623.857119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076624.049115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076624.561121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076624.561130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076624.561176:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076624.561273:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076624.881248:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076625.073177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076625.585226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076625.585234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076625.585303:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076625.585351:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076625.905208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076626.097179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076626.609172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076626.609181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076626.609270:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076626.609353:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076626.929182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076626.929183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076626.929193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076626.929198:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076626.929209:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2332 00000100:00080000:1.0:1769076626.929240:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076626.929253:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076626.929316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913324544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076626.929366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076626.929403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076626.929405:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076626.929410:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2332 00000100:00080000:1.0:1769076626.929414:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076626.929418:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076626.929429:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913324544:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769076626.929431:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076626.929449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913324672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076626.929458:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076626.929461:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076626.929462:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076626.929472:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2332 00000100:00080000:1.0:1769076626.929475:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076626.929480:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076626.929484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076626.929493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913324800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076626.929499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076626.929502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076626.929503:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076626.929505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2332 00000100:00080000:1.0:1769076626.929508:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076626.929511:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076626.929513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076626.929523:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913324928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076626.929527:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076626.929539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076626.929543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076626.929546:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076626.929552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2332 00000100:00080000:1.0:1769076626.929556:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076626.929560:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076626.929567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076626.929573:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9350100c8a80 x1855008913324544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076626.929584:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913325056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076626.929595:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913324544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076626.929607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076626.929611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076626.929612:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076626.929613:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076626.929622:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076626.929623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2332 00000100:00080000:1.0:1769076626.929630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076626.929632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913324544:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076626.929637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076626.929645:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076626.929657:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913325184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076626.929665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076626.929668:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324672:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076626.929670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076626.929675:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076626.929678:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2332 00000100:00080000:1.0:1769076626.929681:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076626.929683:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076626.929687:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076626.929713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076626.929717:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913325312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076626.929730:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9350100c9c00 x1855008913324672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.929742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913324672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.929749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.929751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076626.929753:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076626.929774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076626.929792:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076626.929800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9350100c9180 x1855008913324800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.929810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913324800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.929816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.929819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076626.929821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076626.929826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076626.929838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076626.929845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9350100cb100 x1855008913324928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.929855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913324928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.929862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.929864:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076626.929867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913324928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076626.929889:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325056:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076626.929906:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076626.929915:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9350100c8000 x1855008913325056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.929927:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913325056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.929960:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.929963:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076626.929965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076626.929979:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076626.929993:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076626.930000:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9350100cad80 x1855008913325184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.930008:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913325184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.930018:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.930021:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076626.930023:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076626.930027:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076626.930036:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076626.930042:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9350100c9f80 x1855008913325312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.930087:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913325312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.930094:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.930096:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076626.930098:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076627.121131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076627.633201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076627.633213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076627.633281:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076627.633312:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076627.953139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076628.145205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076628.657168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076628.657189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076628.657221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076628.657242:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076628.977184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076629.169168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076629.681170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076629.681170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076629.681170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076629.681207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076630.001264:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076630.193120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076630.705155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076630.705156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076630.705165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076630.705229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076631.025173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076631.217113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076631.729235:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076631.729243:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076631.729244:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076631.729349:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076632.049146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076632.049153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076632.049157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076632.049163:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076632.049172:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2337 00000100:00080000:1.0:1769076632.049184:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076632.049192:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076632.049246:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913325440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076632.049270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076632.049273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076632.049274:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076632.049277:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2337 00000100:00080000:1.0:1769076632.049280:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076632.049296:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076632.049307:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076632.049318:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913325440:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076632.049325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913325568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076632.049335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076632.049338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076632.049340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076632.049343:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2337 00000100:00080000:1.0:1769076632.049346:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076632.049348:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076632.049352:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076632.049363:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913325696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076632.049370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076632.049372:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076632.049373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076632.049376:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2337 00000100:00080000:1.0:1769076632.049379:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076632.049381:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076632.049384:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076632.049390:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076632.049395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913325824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076632.049403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076632.049406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076632.049407:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076632.049410:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2337 00000100:00080000:1.0:1769076632.049413:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076632.049415:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076632.049418:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076632.049428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913325952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076632.049434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076632.049438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076632.049440:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076632/real 1769076632] req@ffff9350100c8380 x1855008913325440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076632.049440:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076632.049447:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2337 00000100:00080000:1.0:1769076632.049453:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076632.049459:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076632.049464:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076632.049479:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913325440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076632.049479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913326080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076632.049513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076632.049516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076632.049518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076632.049524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076632.049534:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076632.049539:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2337 00000100:00100000:0.0:1769076632.049541:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913325440:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076632.049548:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076632.049554:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076632.049559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076632.049569:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913325568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076632.049571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913326208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076632.049598:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076632.049610:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076632/real 1769076632] req@ffff9350100ca300 x1855008913325568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076632.049619:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913325568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076632.049626:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076632.049629:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076632.049631:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913325568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076632.049636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913325696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076632.049648:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076632.049654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076632/real 1769076632] req@ffff9350100ca680 x1855008913325696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076632.049667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913325696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076632.049672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076632.049674:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076632.049676:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913325696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076632.049680:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913325824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076632.049690:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076632.049695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076632/real 1769076632] req@ffff9350100c8e00 x1855008913325824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076632.049703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913325824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076632.049708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076632.049710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076632.049712:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913325824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076632.049715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076632.049724:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076632.049730:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076632/real 1769076632] req@ffff9350100c9880 x1855008913325952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076632.049737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913325952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076632.049747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076632.049749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076632.049752:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913325952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076632.049755:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076632.049764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076632.049770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076632/real 1769076632] req@ffff9350100c9500 x1855008913326080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076632.049776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913326080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076632.049781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076632.049783:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076632.049785:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076632.049796:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076632.049807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076632.049814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076632/real 1769076632] req@ffff9350100caa00 x1855008913326208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076632.049822:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913326208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076632.049826:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076632.049829:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076632.049833:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076632.241167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076632.753194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076632.753216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076632.753228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076632.753426:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076633.073150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076633.265123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076633.777223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076633.777276:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076633.777286:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076633.777370:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076634.097175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076634.289176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076634.801211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076634.801221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076634.801225:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076634.801319:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076635.121280:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076635.313268:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076635.825125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076635.825138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076635.825181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076635.825207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076636.145297:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076636.337257:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076636.849217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076636.849218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076636.849258:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076636.849290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076637.169171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076637.169254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076637.169262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076637.169267:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076637.169277:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2342 00000100:00080000:2.0:1769076637.169302:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076637.169309:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076637.169363:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913326336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076637.169390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076637.169394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076637.169395:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076637.169398:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2342 00000100:00080000:2.0:1769076637.169401:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076637.169404:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076637.169426:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076637.169439:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913326464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076637.169440:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913326336:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076637.169455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076637.169458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076637.169459:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076637.169463:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2342 00000100:00080000:2.0:1769076637.169465:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076637.169468:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076637.169472:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076637.169515:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913326592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076637.169522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076637.169525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076637.169526:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076637.169529:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2342 00000100:00080000:2.0:1769076637.169532:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076637.169535:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076637.169539:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076637.169544:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076637.169548:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913326720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076637.169563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076637.169566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076637.169567:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076637.169570:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2342 00000100:00080000:2.0:1769076637.169573:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076637.169575:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076637.169579:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076637.169588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913326848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076637.169594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076637.169596:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076637.169598:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076637.169601:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076637/real 1769076637] req@ffff935002a61880 x1855008913326336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076637.169604:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2342 00000100:00080000:2.0:1769076637.169610:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076637.169624:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076637.169628:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076637.169632:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913326336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076637.169640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913326976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076637.169644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076637.169653:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076637.169654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076637.169660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076637.169663:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913326336:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076637.169664:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076637.169672:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2342 00000100:00080000:2.0:1769076637.169675:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076637.169677:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076637.169680:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076637.169686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913327104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076637.169694:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913326464:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076637.169713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076637.169722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076637/real 1769076637] req@ffff935002a62a00 x1855008913326464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076637.169730:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913326464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076637.169737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076637.169739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076637.169742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913326464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076637.169746:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913326592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076637.169757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076637.169762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076637/real 1769076637] req@ffff935002a60000 x1855008913326592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076637.169769:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913326592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076637.169774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076637.169776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076637.169778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913326592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076637.169781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913326720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076637.169791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076637.169797:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076637/real 1769076637] req@ffff935002a60a80 x1855008913326720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076637.169804:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913326720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076637.169809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076637.169811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076637.169813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913326720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076637.169817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326848:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076637.169837:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076637.169843:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076637/real 1769076637] req@ffff935002a62680 x1855008913326848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076637.169850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913326848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076637.169855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076637.169857:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076637.169859:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076637.169862:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076637.169871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076637.169877:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076637/real 1769076637] req@ffff935002a62300 x1855008913326976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076637.169884:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913326976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076637.169889:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076637.169891:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076637.169892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913326976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076637.169910:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913327104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076637.169919:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076637.169934:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076637/real 1769076637] req@ffff935002a61500 x1855008913327104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076637.169942:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913327104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076637.169947:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076637.169949:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076637.169951:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913327104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076637.361179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076637.873231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076637.873262:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076637.873262:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076637.873317:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076638.193292:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076638.385239:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076638.897234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076638.897239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076638.897244:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076638.897298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076639.217196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076639.409129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076639.921185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076639.921201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076639.921302:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076639.921303:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076640.241393:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076640.433188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076640.945160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076640.945161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076640.945168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076640.945355:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076641.265243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076641.457119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076641.969190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076641.969190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076641.969190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076641.969323:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076642.289207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076642.289253:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076642.289265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076642.289271:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076642.289284:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2347 00000100:00080000:2.0:1769076642.289299:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076642.289319:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076642.289371:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913327232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076642.289409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076642.289414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076642.289416:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076642.289421:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2347 00000100:00080000:2.0:1769076642.289425:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076642.289429:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076642.289441:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076642.289466:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913327360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076642.289478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076642.289482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076642.289484:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076642.289489:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2347 00000100:00080000:2.0:1769076642.289494:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076642.289515:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913327232:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076642.289531:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076642.289538:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076642.289554:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913327488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076642.289567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076642.289571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076642.289574:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076642.289579:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2347 00000100:00080000:2.0:1769076642.289583:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076642.289587:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076642.289592:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076642.289604:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913327616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076642.289613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076642.289617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076642.289619:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076642.289624:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2347 00000100:00080000:2.0:1769076642.289627:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076642.289631:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076642.289636:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076642.289638:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076642.289652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913327744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076642.289663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076642.289667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076642.289669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076642.289673:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2347 00000100:00080000:2.0:1769076642.289678:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076642.289682:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076642.289687:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076642.289703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913327872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076642.289712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076642.289716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076642.289719:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076642.289725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076642/real 1769076642] req@ffff935002a63100 x1855008913327232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076642.289726:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2347 00000100:00080000:2.0:1769076642.289734:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076642.289741:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076642.289749:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076642.289774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913328000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076642.289807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913327232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076642.289823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076642.289830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076642.289862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913327232:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076642.289875:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913327360:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076642.289926:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076642.289941:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076642/real 1769076642] req@ffff935002a61f80 x1855008913327360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076642.289955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913327360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076642.289964:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076642.289968:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076642.289972:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913327360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076642.290002:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913327488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076642.290037:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076642.290049:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076642/real 1769076642] req@ffff935002a61c00 x1855008913327488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076642.290064:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913327488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076642.290075:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076642.290139:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076642.290144:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913327488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076642.290161:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913327616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076642.290183:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076642.290195:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076642/real 1769076642] req@ffff935002a61180 x1855008913327616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076642.290210:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913327616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076642.290219:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076642.290222:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076642.290226:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913327616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076642.290234:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913327744:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076642.290255:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076642.290268:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076642/real 1769076642] req@ffff935002a61500 x1855008913327744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076642.290282:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913327744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076642.290289:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076642.290291:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076642.290294:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913327744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076642.290315:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913327872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076642.290334:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076642.290342:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076642/real 1769076642] req@ffff935002a62300 x1855008913327872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076642.290351:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913327872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076642.290364:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076642.290367:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076642.290369:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913327872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076642.290374:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076642.290387:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076642.290394:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076642/real 1769076642] req@ffff935002a62680 x1855008913328000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076642.290402:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913328000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076642.290409:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076642.290411:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076642.290413:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076642.481148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076642.993157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076642.993184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076642.993186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076642.993189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076643.313162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076643.505164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076644.017196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076644.017240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076644.017270:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076644.017361:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076644.337231:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076644.529207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076645.041207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076645.041207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076645.041215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076645.041275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076645.361133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076645.553160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076646.065161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076646.065162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076646.065194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076646.065303:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076646.385139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076646.577136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076647.089127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076647.089128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076647.089165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076647.089210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076647.409137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076647.409142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076647.409150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076647.409154:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076647.409163:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2353 00000100:00080000:2.0:1769076647.409175:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076647.409181:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076647.409219:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913328128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076647.409243:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076647.409246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076647.409248:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076647.409251:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2353 00000100:00080000:2.0:1769076647.409254:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076647.409257:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076647.409267:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076647.409277:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913328128:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076647.409278:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913328256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076647.409291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076647.409293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076647.409294:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076647.409297:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2353 00000100:00080000:2.0:1769076647.409300:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076647.409302:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076647.409306:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076647.409313:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913328384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076647.409320:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076647.409322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076647.409324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076647.409326:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2353 00000100:00080000:2.0:1769076647.409329:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076647.409331:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076647.409334:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076647.409350:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076647.409356:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913328512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076647.409367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076647.409371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076647.409373:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076647.409378:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2353 00000100:00080000:2.0:1769076647.409382:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076647.409388:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076647.409391:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076647/real 1769076647] req@ffff935002a60a80 x1855008913328128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076647.409395:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076647.409412:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913328128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076647.409416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913328640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076647.409436:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076647.409437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076647.409443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076647.409445:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076647.409446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076647.409455:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913328128:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076647.409455:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2353 00000100:00080000:2.0:1769076647.409462:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076647.409467:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076647.409470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076647.409477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913328768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076647.409483:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913328256:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076647.409487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076647.409491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076647.409492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076647.409497:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2353 00000100:00080000:2.0:1769076647.409501:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076647.409504:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076647.409508:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769076647.409511:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076647.409521:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076647/real 1769076647] req@ffff935002a60000 x1855008913328256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076647.409530:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913328896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076647.409534:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913328256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076647.409553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076647.409557:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076647.409560:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913328256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076647.409566:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913328384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076647.409580:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076647.409586:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076647/real 1769076647] req@ffff935002a62a00 x1855008913328384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076647.409593:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913328384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076647.409603:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076647.409605:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076647.409617:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913328384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076647.409624:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913328512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076647.409642:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076647.409654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076647/real 1769076647] req@ffff935002a61880 x1855008913328512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076647.409667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913328512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076647.409675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076647.409679:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076647.409682:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913328512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076647.409687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328640:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076647.409702:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076647.409710:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076647/real 1769076647] req@ffff935002a62680 x1855008913328640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076647.409718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913328640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076647.409724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076647.409726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076647.409728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076647.409741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076647.409752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076647.409758:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076647/real 1769076647] req@ffff935002a62300 x1855008913328768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076647.409765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913328768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076647.409770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076647.409772:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076647.409774:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076647.409777:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076647.409786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076647.409792:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076647/real 1769076647] req@ffff935002a61500 x1855008913328896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076647.409799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913328896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076647.409804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076647.409806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076647.409808:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913328896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076647.601138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076648.113123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076648.113124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076648.113142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076648.113179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076648.433116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076648.625120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076649.137135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076649.137135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076649.137149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076649.137195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076649.457126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076649.649150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076650.161142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076650.161148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076650.161187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076650.161219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076650.481162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076650.673106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076651.185127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076651.185127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076651.185153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076651.185183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076651.505131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076651.697112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076652.209162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076652.209162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076652.209164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076652.209282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076652.529150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076652.529186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076652.529197:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076652.529203:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076652.529214:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2358 00000100:00080000:2.0:1769076652.529226:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076652.529236:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076652.529278:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913329024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076652.529310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076652.529314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076652.529317:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076652.529322:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2358 00000100:00080000:2.0:1769076652.529328:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076652.529333:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076652.529345:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076652.529350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913329024:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076652.529368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913329152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076652.529383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076652.529387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076652.529390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076652.529395:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2358 00000100:00080000:2.0:1769076652.529401:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076652.529405:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076652.529412:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076652.529430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913329280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076652.529441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076652.529445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076652.529447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076652.529454:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2358 00000100:00080000:2.0:1769076652.529459:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076652.529463:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076652.529469:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913329152:192.168.204.121@tcp:38:kworker.0 00000008:00100000:2.0:1769076652.529470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076652.529499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913329408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076652.529514:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913329280:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076652.529514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076652.529530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076652.529544:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076652.529555:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2358 00000100:00080000:2.0:1769076652.529561:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076652.529562:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913329408:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076652.529577:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076652.529586:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076652.529606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913329536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076652.529628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076652.529633:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076652.529636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076652.529645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2358 00000100:00100000:0.0:1769076652.529650:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913329536:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076652.529653:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076652.529661:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076652.529669:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076652.529686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913329664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076652.529713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076652.529717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076652.529719:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076652.529730:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2358 00000100:00080000:2.0:1769076652.529736:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076652.529741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913329664:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076652.529745:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076652.529755:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076652.529778:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913329792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076652.529815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913329792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076652.721111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076653.233116:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076653.233132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076653.233133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076653.233168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076653.553146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076653.745140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076654.257143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076654.257143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076654.257143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076654.257178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076654.577126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076654.769104:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076655.281143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076655.281144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076655.281179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076655.281257:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076655.601139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769076655.601307:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769076655.601345:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076655.601363:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076655.601403:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076655.601413:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076655.601425:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076655.601437:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076655.601452:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076655.601460:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076652/real 1769076655] req@ffff935002a63100 x1855008913329408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769076655.601463:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076655.601493:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913329408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076655.601506:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076655.601513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076655.601517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913329408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076655.601526:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076652/real 1769076655] req@ffff935002a61c00 x1855008913329152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076655.601540:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913329152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076655.601550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076655.601555:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076655.601559:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913329152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076655.601566:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076652/real 1769076655] req@ffff935002a61f80 x1855008913329280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076655.601580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913329280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076655.601588:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076655.601593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076655.601597:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913329280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076655.601606:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076652/real 1769076655] req@ffff935002a61180 x1855008913329024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076655.601619:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913329024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076655.601627:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076655.601631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076655.601634:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913329024:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076655.601676:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076652/real 1769076655] req@ffff935002a62680 x1855008913329792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076655.601693:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913329792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076655.601703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076655.601707:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076655.601711:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913329792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076655.601719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076652/real 1769076655] req@ffff935002a62300 x1855008913329664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076655.601731:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913329664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076655.601740:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076655.601744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076655.601747:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913329664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076655.601755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076652/real 1769076655] req@ffff935002a61500 x1855008913329536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076655.601770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913329536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076655.601780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076655.601784:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076655.601788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913329536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076655.793130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076656.305159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076656.305159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076656.305165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076656.305258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076656.625126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076656.817125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076657.329143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076657.329143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076657.329165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076657.329192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076657.649174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076657.649310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076657.649317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076657.649321:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076657.649330:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2363 00000100:00080000:3.0:1769076657.649340:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076657.649347:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076657.649381:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913329920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076657.649403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076657.649406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076657.649407:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076657.649410:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2363 00000100:00080000:3.0:1769076657.649413:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076657.649416:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076657.649426:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076657.649435:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913330048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076657.649444:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913329920:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076657.649445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076657.649454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076657.649456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076657.649461:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2363 00000100:00080000:3.0:1769076657.649465:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076657.649469:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076657.649475:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076657.649489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913330176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076657.649505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076657.649509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076657.649511:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076657.649515:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2363 00000100:00080000:3.0:1769076657.649530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076657.649535:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076657.649541:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076657.649550:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076657.649565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913330304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076657.649576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076657.649579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076657.649582:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076657.649587:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2363 00000100:00080000:3.0:1769076657.649591:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076657.649594:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076657/real 1769076657] req@ffff935002a7e300 x1855008913329920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076657.649600:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076657.649613:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076657.649625:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913329920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076657.649630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913330432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076657.649646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076657.649650:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076657.649655:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076657.649656:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076657.649658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076657.649665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913329920:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076657.649667:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2363 00000100:00080000:3.0:1769076657.649676:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076657.649682:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076657.649687:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076657.649700:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913330560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076657.649702:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330048:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076657.649726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076657.649731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076657.649735:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076657.649740:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076657.649742:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2363 00000100:00080000:3.0:1769076657.649749:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076657.649753:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076657.649758:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076657.649765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076657/real 1769076657] req@ffff935002a7ca80 x1855008913330048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076657.649773:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913330688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076657.649786:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913330048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076657.649806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076657.649812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076657.649816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076657.649822:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076657.649846:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076657.649890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076657/real 1769076657] req@ffff935002a7c380 x1855008913330176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076657.649905:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913330176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076657.649914:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076657.649918:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076657.649922:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076657.649928:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076657.649954:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076657.649966:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076657/real 1769076657] req@ffff935002a7f100 x1855008913330304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076657.649979:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913330304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076657.649987:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076657.649989:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076657.649993:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076657.649999:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913330432:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076657.650021:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076657.650034:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076657/real 1769076657] req@ffff935002a7f800 x1855008913330432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076657.650046:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913330432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076657.650082:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076657.650085:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076657.650089:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913330432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076657.650117:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913330560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076657.650142:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076657.650154:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076657/real 1769076657] req@ffff935002a7fb80 x1855008913330560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076657.650165:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913330560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076657.650173:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076657.650176:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076657.650179:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913330560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076657.650184:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913330688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076657.650201:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076657.650213:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076657/real 1769076657] req@ffff935002a7c700 x1855008913330688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076657.650224:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913330688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076657.650232:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076657.650235:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076657.650238:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913330688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076657.841134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076658.353170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076658.353170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076658.353171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076658.353206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076658.673158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076658.865122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076659.377128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076659.377128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076659.377134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076659.377175:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076659.697171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076659.889112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076660.401148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076660.401148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076660.401191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076660.401212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076660.721127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076660.913102:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076661.425116:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076661.425126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076661.425135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076661.425292:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076661.745182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076661.937161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076662.449105:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076662.449164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076662.449164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076662.449234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076662.769108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076662.769119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076662.769125:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076662.769134:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2368 00000100:00080000:3.0:1769076662.769146:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076662.769153:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076662.769172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076662.769200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913330816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076662.769226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076662.769229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076662.769230:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076662.769234:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2368 00000100:00080000:3.0:1769076662.769236:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076662.769239:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076662.769249:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076662.769262:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913330944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076662.769270:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913330816:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076662.769292:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076662.769296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076662.769297:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076662.769303:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2368 00000100:00080000:3.0:1769076662.769306:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076662.769308:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076662.769314:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076662.769325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913331072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076662.769338:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076662.769341:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076662.769342:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076662.769345:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2368 00000100:00080000:3.0:1769076662.769348:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076662.769351:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076662.769356:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076662.769366:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913331200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076662.769372:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076662.769382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076662.769385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076662.769388:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076662.769395:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2368 00000100:00080000:3.0:1769076662.769401:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076662.769404:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076662.769409:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076662.769422:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913331328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076662.769428:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076662/real 1769076662] req@ffff935002a7dc00 x1855008913330816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076662.769468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076662.769474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076662.769475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076662.769478:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2368 00000100:00080000:3.0:1769076662.769485:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076662.769490:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076662.769495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076662.769496:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913330816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076662.769508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913331456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076662.769510:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076662.769528:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076662.769531:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076662.769539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076662.769541:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913330816:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076662.769542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076662.769562:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2368 00000100:00080000:3.0:1769076662.769567:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076662.769572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076662.769575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076662.769588:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076662.769589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913331584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076662.769640:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076662.769659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076662/real 1769076662] req@ffff935002a7d180 x1855008913330944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076662.769673:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913330944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076662.769683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076662.769686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076662.769688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913330944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076662.769693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076662.769711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076662.769719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076662/real 1769076662] req@ffff935002a7df80 x1855008913331072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076662.769727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913331072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076662.769732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076662.769735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076662.769737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076662.769740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076662.769750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076662.769757:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076662/real 1769076662] req@ffff935002a7c000 x1855008913331200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076662.769764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913331200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076662.769786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076662.769789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076662.769791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076662.769795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913331328:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076662.769811:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076662.769819:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076662/real 1769076662] req@ffff935002a7c700 x1855008913331328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076662.769827:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913331328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076662.769832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076662.769834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076662.769836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913331328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076662.769840:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913331456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076662.769850:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076662.769856:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076662/real 1769076662] req@ffff935002a7fb80 x1855008913331456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076662.769868:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913331456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076662.769876:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076662.769879:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076662.769892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913331456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076662.769920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913331584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076662.769941:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076662.769950:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076662/real 1769076662] req@ffff935002a7f800 x1855008913331584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076662.769958:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913331584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076662.769964:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076662.769966:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076662.769968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913331584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076662.961177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076663.473151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076663.473180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076663.473192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076663.473245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076663.793155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076663.985124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076664.497133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076664.497135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076664.497138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076664.497171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076664.817123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076665.009128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076665.521136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076665.521142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076665.521145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076665.521249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076665.841142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076666.033128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076666.545190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076666.545190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076666.545259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076666.545275:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076666.865119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076667.057155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076667.569134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076667.569140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076667.569193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076667.569225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076667.889151:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076667.889164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1769076667.889165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076667.889169:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076667.889181:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2373 00000100:00080000:1.0:1769076667.889193:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076667.889201:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076667.889242:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913331712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076667.889269:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076667.889274:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076667.889276:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076667.889282:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2373 00000100:00080000:1.0:1769076667.889286:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076667.889290:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076667.889299:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913331712:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769076667.889309:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076667.889328:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913331840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076667.889341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076667.889345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076667.889347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076667.889350:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2373 00000100:00080000:1.0:1769076667.889355:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076667.889366:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076667.889371:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769076667.889376:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076667.889391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913331968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076667.889409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076667.889412:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff9350100cb800 x1855008913331712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076667.889414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076667.889418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076667.889427:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2373 00000100:00080000:1.0:1769076667.889432:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076667.889435:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913331712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076667.889446:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076667.889446:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076667.889455:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1769076667.889456:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076667.889463:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913331712:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076667.889474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913332096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076667.889487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076667.889488:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331840:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076667.889497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076667.889506:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076667.889513:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2373 00000800:00000100:0.0:1769076667.889515:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076667.889521:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076667.889528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076667.889531:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff9350100caa00 x1855008913331840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076667.889535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076667.889553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913332224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076667.889556:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913331840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076667.889579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076667.889583:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076667.889590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076667.889591:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076667.889596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076667.889601:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331840:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076667.889605:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2373 00000100:00080000:1.0:1769076667.889612:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076667.889617:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331968:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769076667.889620:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076667.889629:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076667.889649:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913332352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076667.889650:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076667.889664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076667.889668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076667.889670:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076667.889672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff9350100c9500 x1855008913331968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076667.889677:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2373 00000100:00080000:1.0:1769076667.889696:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076667.889704:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076667.889710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913331968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076667.889711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076667.889722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076667.889729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769076667.889731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913332480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076667.889734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913331968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076667.889753:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076667.889775:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076667.889788:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff9350100c9880 x1855008913332096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076667.889801:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913332096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076667.889810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076667.889814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076667.889817:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076667.889824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913332224:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076667.889845:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076667.889856:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff9350100c8e00 x1855008913332224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076667.889865:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913332224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076667.889876:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076667.889878:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076667.889881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913332224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076667.889885:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913332352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076667.889898:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076667.889904:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff9350100ca680 x1855008913332352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076667.889912:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913332352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076667.889917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076667.889919:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076667.889921:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913332352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076667.889924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913332480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076667.889934:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076667.889940:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff9350100ca300 x1855008913332480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076667.889947:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913332480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076667.889952:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076667.889954:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076667.889956:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913332480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076668.081120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076668.593141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076668.593167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076668.593195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076668.593232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076668.913167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076669.105152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076669.617133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076669.617164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076669.617188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076669.617189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076669.937201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076670.129101:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076670.641149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076670.641149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076670.641191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076670.641251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076670.961128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076671.153104:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076671.665144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076671.665154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076671.665182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076671.665228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076671.985213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076672.177154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076672.689123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076672.689123:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076672.689130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076672.689159:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076673.009166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076673.009167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076673.009178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076673.009183:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076673.009191:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2378 00000100:00080000:2.0:1769076673.009209:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076673.009220:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076673.009257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913332608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076673.009279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076673.009282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076673.009283:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076673.009286:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2378 00000100:00080000:2.0:1769076673.009289:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076673.009292:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076673.009301:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076673.009314:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913332608:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076673.009314:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913332736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076673.009333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076673.009336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076673.009337:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076673.009340:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2378 00000100:00080000:2.0:1769076673.009343:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076673.009345:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076673.009349:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076673.009359:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913332864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076673.009366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076673.009368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076673.009369:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076673.009371:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2378 00000100:00080000:2.0:1769076673.009374:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076673.009377:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076673.009380:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076673.009388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913332992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076673.009394:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076673.009403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076673.009406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076673.009407:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076673.009412:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2378 00000100:00080000:2.0:1769076673.009415:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076673.009418:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076673.009421:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076673.009431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913333120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076673.009437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076673.009440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076673.009442:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076673.009444:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076673/real 1769076673] req@ffff935002a61880 x1855008913332608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076673.009447:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2378 00000100:00080000:2.0:1769076673.009453:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076673.009459:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076673.009463:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076673.009470:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913332608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076673.009475:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913333248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076673.009488:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076673.009494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076673.009497:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076673.009501:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076673.009506:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913332608:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076673.009507:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076673.009518:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2378 00000100:00080000:2.0:1769076673.009524:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076673.009529:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076673.009536:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076673.009542:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076673.009550:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913333376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076673.009575:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076673.009590:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076673/real 1769076673] req@ffff935002a62a00 x1855008913332736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076673.009631:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913332736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076673.009638:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076673.009640:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076673.009643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076673.009648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076673.009667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076673.009679:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076673/real 1769076673] req@ffff935002a60000 x1855008913332864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076673.009692:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913332864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076673.009701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076673.009705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076673.009708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076673.009715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076673.009735:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076673.009745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076673/real 1769076673] req@ffff935002a60a80 x1855008913332992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076673.009753:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913332992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076673.009763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076673.009766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076673.009768:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913332992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076673.009772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913333120:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076673.009784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076673.009790:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076673/real 1769076673] req@ffff935002a61500 x1855008913333120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076673.009797:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913333120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076673.009804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076673.009805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076673.009807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913333120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076673.009811:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913333248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076673.009821:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076673.009827:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076673/real 1769076673] req@ffff935002a62300 x1855008913333248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076673.009834:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913333248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076673.009839:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076673.009841:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076673.009844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913333248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076673.009859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913333376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076673.009869:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076673.009875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076673/real 1769076673] req@ffff935002a62680 x1855008913333376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076673.009883:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913333376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076673.009888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076673.009890:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076673.009892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913333376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076673.201123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076673.713189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076673.713189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076673.713198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076673.713268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076674.033147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076674.225138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076674.737162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076674.737166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076674.737203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076674.737282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076675.057209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076675.249143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076675.761163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076675.761206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076675.761219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076675.761262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076676.081208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076676.273210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076676.785148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076676.785150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076676.785153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076676.785222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076677.105186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076677.297184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076677.809184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076677.809205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076677.809210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076677.809304:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076678.129106:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076678.129125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076678.129135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076678.129139:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076678.129147:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2383 00000100:00080000:2.0:1769076678.129159:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076678.129166:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076678.129223:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913333504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076678.129252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076678.129256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076678.129257:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076678.129272:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2383 00000100:00080000:2.0:1769076678.129276:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076678.129280:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076678.129289:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076678.129297:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913333504:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076678.129304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913333632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076678.129313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076678.129316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076678.129319:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076678.129322:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2383 00000100:00080000:2.0:1769076678.129325:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076678.129346:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076678.129353:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076678.129368:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913333760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076678.129405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076678.129409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076678.129412:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076678.129417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2383 00000100:00080000:2.0:1769076678.129422:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076678.129426:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076678.129432:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076678.129439:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076678.129455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913333888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076678.129474:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076678.129477:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076678.129479:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076678.129483:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2383 00000100:00080000:2.0:1769076678.129487:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076678.129492:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076678/real 1769076678] req@ffff935002a61180 x1855008913333504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076678.129493:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076678.129500:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076678.129557:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913334016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076678.129564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076678.129566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076678.129569:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076678.129576:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2383 00000100:00080000:2.0:1769076678.129582:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076678.129589:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076678.129593:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076678.129601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913333504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076678.129604:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913334144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076678.129626:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076678.129626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076678.129630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076678.129634:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076678.129638:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2383 00000100:00080000:2.0:1769076678.129640:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076678.129642:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076678.129646:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769076678.129646:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076678.129657:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913333504:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076678.129666:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913334272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076678.129689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913333632:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076678.129711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076678.129722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076678/real 1769076678] req@ffff935002a61f80 x1855008913333632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076678.129731:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913333632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076678.129738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076678.129741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076678.129743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913333632:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076678.129748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913333760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076678.129759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076678.129766:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076678/real 1769076678] req@ffff935002a61c00 x1855008913333760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076678.129774:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913333760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076678.129784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076678.129786:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076678.129789:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913333760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076678.129792:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913333888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076678.129801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076678.129806:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076678/real 1769076678] req@ffff935002a63100 x1855008913333888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076678.129814:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913333888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076678.129819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076678.129821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076678.129823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913333888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076678.129826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334016:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076678.129836:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076678.129842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076678/real 1769076678] req@ffff935002a62680 x1855008913334016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076678.129849:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913334016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076678.129873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076678.129875:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076678.129878:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076678.129883:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076678.129894:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076678.129901:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076678/real 1769076678] req@ffff935002a62300 x1855008913334144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076678.129908:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913334144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076678.129914:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076678.129916:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076678.129918:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076678.129921:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076678.129931:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076678.129937:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076678/real 1769076678] req@ffff935002a61500 x1855008913334272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076678.129945:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913334272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076678.129951:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076678.129952:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076678.129954:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076678.321132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076678.833134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076678.833159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076678.833199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076678.833202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076679.153202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076679.345179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076679.857179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076679.857181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076679.857183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076679.857229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076680.177147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076680.369196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076680.881174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076680.881201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076680.881203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076680.881293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076681.201172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076681.393200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076681.905201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076681.905201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076681.905269:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076681.905324:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076682.225139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076682.417154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076682.929212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076682.929212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076682.929235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076682.929251:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769076683.249223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076683.249236:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1769076683.249244:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076683.249244:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076683.249268:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2388 00000100:00080000:1.0:1769076683.249287:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076683.249296:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076683.249345:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913334400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076683.249383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076683.249388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076683.249390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076683.249396:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2388 00000100:00080000:1.0:1769076683.249400:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076683.249409:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076683.249457:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913334400:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769076683.249483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076683.249519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913334528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076683.249545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076683.249551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076683.249557:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076683.249565:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2388 00000100:00080000:1.0:1769076683.249575:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076683.249583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076683.249592:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076683.249601:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076683.249619:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913334656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076683.249636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076683.249640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076683.249642:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076683.249648:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2388 00000100:00080000:1.0:1769076683.249654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076683.249662:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076683.249668:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076683.249676:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076683/real 1769076683] req@ffff9350100c8380 x1855008913334400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076683.249684:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913334784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076683.249743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076683.249779:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769076683.249781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913334400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076683.249784:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076683.249809:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2388 00000100:00080000:0.0:1769076683.249823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076683.249823:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076683.249846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076683.249848:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076683.249863:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913334400:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769076683.249863:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076683.249914:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913334912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076683.249927:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913334528:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076683.249979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076683.249983:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076683.249986:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076683.249992:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2388 00000100:00080000:1.0:1769076683.249998:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076683.250004:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076683.250011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076683.250021:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076683.250026:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076683/real 1769076683] req@ffff9350100c9f80 x1855008913334528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076683.250056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913335040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076683.250067:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913334528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076683.250074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076683.250087:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076683.250119:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076683.250125:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913334528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076683.250133:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913334656:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769076683.250144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076683.250149:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076683.250157:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2388 00000100:00080000:1.0:1769076683.250164:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076683.250171:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076683.250176:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076683.250185:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076683.250188:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076683/real 1769076683] req@ffff9350100cad80 x1855008913334656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076683.250206:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913335168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076683.250211:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913334656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076683.250275:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076683.250298:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076683.250306:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913334656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076683.250317:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913334784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076683.250337:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076683.250345:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076683/real 1769076683] req@ffff9350100c8000 x1855008913334784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076683.250356:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913334784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076683.250363:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076683.250366:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076683.250368:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913334784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076683.250386:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334912:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076683.250400:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076683.250410:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076683/real 1769076683] req@ffff9350100cb100 x1855008913334912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076683.250420:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913334912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076683.250453:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076683.250457:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076683.250460:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913334912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076683.250467:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076683.250489:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076683.250507:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076683/real 1769076683] req@ffff9350100c9180 x1855008913335040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076683.250524:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913335040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076683.250536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076683.250541:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076683.250545:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076683.250554:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076683.250582:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076683.250597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076683/real 1769076683] req@ffff9350100c9c00 x1855008913335168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076683.250609:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913335168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076683.250616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076683.250620:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076683.250623:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076683.441174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076683.953141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076683.953196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076683.953207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076683.953254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076684.273172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076684.465186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076684.977139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076684.977242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076684.977295:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076684.977299:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076685.297220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076685.489148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076686.001195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076686.001253:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076686.001254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076686.001267:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076686.321187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076686.513152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076687.025221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076687.025245:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076687.025274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076687.025297:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076687.345240:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076687.537170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076688.049138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076688.049138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076688.049155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076688.049183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076688.369142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076688.369156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1769076688.369158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076688.369161:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076688.369174:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2394 00000100:00080000:1.0:1769076688.369189:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076688.369197:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076688.369239:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913335296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076688.369273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076688.369277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076688.369279:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076688.369283:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2394 00000100:00080000:1.0:1769076688.369286:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076688.369290:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076688.369300:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076688.369306:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913335296:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076688.369316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913335424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076688.369333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076688.369335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076688.369336:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076688.369356:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2394 00000100:00080000:1.0:1769076688.369359:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076688.369363:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076688.369371:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076688.369384:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913335552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076688.369407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076688.369411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076688.369413:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076688.369420:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076688.369422:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2394 00000100:00080000:1.0:1769076688.369429:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076688.369434:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076688.369447:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076688.369459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913335680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076688.369467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076688.369472:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076688/real 1769076688] req@ffff9350100c8a80 x1855008913335296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076688.369472:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076688.369474:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076688.369482:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2394 00000100:00080000:1.0:1769076688.369492:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076688.369495:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076688.369500:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913335296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076688.369501:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076688.369514:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076688.369516:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913335808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076688.369521:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076688.369529:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913335296:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076688.369531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076688.369536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076688.369539:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076688.369542:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2394 00000100:00080000:1.0:1769076688.369544:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076688.369546:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076688.369549:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076688.369579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913335424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076688.369591:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913335936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076688.369605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076688.369609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076688.369612:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076688.369616:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2394 00000100:00080000:1.0:1769076688.369621:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076688.369643:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076688.369645:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076688.369651:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076688.369661:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076688/real 1769076688] req@ffff9350100c9c00 x1855008913335424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076688.369664:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913336064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076688.369703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913335424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076688.369714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076688.369718:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076688.369722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913335424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076688.369730:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913335552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076688.369753:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076688.369765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076688/real 1769076688] req@ffff9350100c9180 x1855008913335552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076688.369787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913335552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076688.369793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076688.369795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076688.369797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913335552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076688.369801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913335680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076688.369814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076688.369821:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076688/real 1769076688] req@ffff9350100cb100 x1855008913335680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076688.369828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913335680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076688.369835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076688.369837:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076688.369839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913335680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076688.369842:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335808:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076688.369851:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076688.369857:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076688/real 1769076688] req@ffff9350100c8000 x1855008913335808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076688.369864:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913335808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076688.369870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076688.369872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076688.369874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076688.369888:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076688.369899:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076688.369905:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076688/real 1769076688] req@ffff9350100cad80 x1855008913335936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076688.369913:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913335936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076688.369918:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076688.369919:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076688.369922:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913335936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076688.369926:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076688.369935:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076688.369941:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076688/real 1769076688] req@ffff9350100c9f80 x1855008913336064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076688.369948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913336064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076688.369954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076688.369955:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076688.369957:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076688.561120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076689.073169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076689.073171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076689.073250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076689.073274:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076689.393190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076689.585153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076690.097222:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076690.097222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076690.097225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076690.097278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076690.417242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076690.609137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076691.121125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076691.121130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076691.121169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076691.121180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076691.441131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076691.633125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076692.145216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076692.145256:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076692.145336:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076692.145375:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076692.465174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076692.657179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076693.169221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076693.169232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076693.169237:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076693.169299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076693.489194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076693.489290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076693.489305:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076693.489313:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076693.489329:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2399 00000100:00080000:2.0:1769076693.489347:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076693.489359:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076693.489421:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913336192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076693.489472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076693.489481:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076693.489484:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076693.489490:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2399 00000100:00080000:2.0:1769076693.489496:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076693.489504:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076693.489522:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076693.489546:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913336192:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076693.489559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913336320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076693.489589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076693.489597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076693.489603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076693.489611:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2399 00000100:00080000:2.0:1769076693.489621:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076693.489627:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076693.489636:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076693.489656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076693.489660:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913336448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076693.489677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076693.489682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076693.489685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076693.489691:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2399 00000100:00080000:2.0:1769076693.489696:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076693.489700:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076693.489707:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076693.489725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076693/real 1769076693] req@ffff935002a60a80 x1855008913336192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076693.489729:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913336576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076693.489780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076693.489785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076693.489790:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076693.489793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913336192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076693.489801:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2399 00000100:00080000:2.0:1769076693.489812:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076693.489823:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076693.489829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769076693.489831:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076693.489849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076693.489859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913336704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076693.489870:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913336192:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076693.489895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076693.489900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076693.489903:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076693.489911:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2399 00000100:00080000:2.0:1769076693.489916:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076693.489920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913336320:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076693.489925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076693.489939:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076693.489961:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913336832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076693.489976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076693.489981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076693.489989:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076693.489990:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076693.490000:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2399 00000100:00080000:2.0:1769076693.490009:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076693.490015:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076693.490022:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076693.490028:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076693/real 1769076693] req@ffff935002a60000 x1855008913336320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076693.490090:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913336320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076693.490105:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076693.490110:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076693.490116:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913336320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076693.490118:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913336960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076693.490132:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913336448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076693.490177:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076693.490200:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076693/real 1769076693] req@ffff935002a62a00 x1855008913336448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076693.490226:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913336448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076693.490236:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076693.490240:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076693.490244:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913336448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076693.490251:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913336576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076693.490313:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076693.490328:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076693/real 1769076693] req@ffff935002a61880 x1855008913336576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076693.490343:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913336576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076693.490354:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076693.490358:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076693.490362:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913336576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076693.490369:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336704:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076693.490389:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076693.490401:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076693/real 1769076693] req@ffff935002a61500 x1855008913336704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076693.490423:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913336704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076693.490451:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076693.490458:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076693.490466:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076693.490505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076693.490544:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076693.490561:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076693/real 1769076693] req@ffff935002a62300 x1855008913336832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076693.490577:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913336832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076693.490588:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076693.490592:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076693.490596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076693.490603:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076693.490622:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076693.490634:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076693/real 1769076693] req@ffff935002a62680 x1855008913336960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076693.490648:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913336960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076693.490657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076693.490660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076693.490665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913336960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076693.681203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076694.193173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076694.193174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076694.193182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076694.193271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076694.513152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076694.705199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076695.217179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076695.217181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076695.217199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076695.217236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076695.537159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076695.729169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076696.241126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076696.241126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076696.241134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076696.241354:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076696.561172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076696.753123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076697.265148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076697.265148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076697.265148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076697.265242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076697.585177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076697.777155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076698.289144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076698.289145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076698.289154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076698.289244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076698.609187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076698.609228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076698.609240:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076698.609247:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076698.609262:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2404 00000100:00080000:2.0:1769076698.609277:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076698.609286:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076698.609340:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913337088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076698.609381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076698.609386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076698.609389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076698.609394:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2404 00000100:00080000:2.0:1769076698.609399:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076698.609405:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076698.609421:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076698.609442:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913337088:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076698.609443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913337216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076698.609470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076698.609474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076698.609477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076698.609482:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2404 00000100:00080000:2.0:1769076698.609487:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076698.609491:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076698.609500:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076698.609518:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913337344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076698.609531:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076698.609536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076698.609539:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076698.609543:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2404 00000100:00080000:2.0:1769076698.609549:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076698.609557:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076698.609562:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076698.609572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076698.609589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913337472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076698.609601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076698.609605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076698.609607:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076698.609614:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2404 00000100:00080000:2.0:1769076698.609620:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076698.609625:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076698.609633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076698/real 1769076698] req@ffff935002a63100 x1855008913337088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076698.609636:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076698.609666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913337600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076698.609676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913337088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076698.609693:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076698.609710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076698.609714:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076698.609716:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076698.609725:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2404 00000100:00080000:2.0:1769076698.609732:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076698.609733:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076698.609745:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076698.609750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913337088:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076698.609754:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076698.609777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913337728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076698.609794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076698.609797:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076698.609799:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076698.609804:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2404 00000100:00080000:2.0:1769076698.609807:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076698.609813:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076698.609822:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913337216:192.168.204.121@tcp:38:kworker.0 00000008:00100000:2.0:1769076698.609824:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076698.609857:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913337856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076698.609899:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076698.609934:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076698/real 1769076698] req@ffff935002a61c00 x1855008913337216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076698.609959:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913337216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076698.609972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076698.609976:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076698.609981:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913337216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076698.609990:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913337344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076698.610019:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076698.610034:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076698/real 1769076698] req@ffff935002a61f80 x1855008913337344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076698.610081:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913337344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076698.610092:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076698.610097:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076698.610101:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913337344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076698.610109:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913337472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076698.610133:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076698.610148:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076698/real 1769076698] req@ffff935002a61180 x1855008913337472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076698.610208:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913337472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076698.610221:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076698.610225:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076698.610229:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913337472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076698.610237:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913337600:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076698.610266:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076698.610281:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076698/real 1769076698] req@ffff935002a62680 x1855008913337600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076698.610295:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913337600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076698.610306:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076698.610309:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076698.610313:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913337600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076698.610336:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913337728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076698.610355:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076698.610367:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076698/real 1769076698] req@ffff935002a62300 x1855008913337728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076698.610381:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913337728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076698.610399:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076698.610403:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076698.610407:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913337728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076698.610413:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913337856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076698.610430:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076698.610441:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076698/real 1769076698] req@ffff935002a61500 x1855008913337856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076698.610455:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913337856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076698.610465:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076698.610468:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076698.610472:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913337856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076698.801211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076699.313262:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076699.313262:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076699.313262:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076699.313329:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076699.633183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076699.825192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076700.337178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076700.337250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076700.337283:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076700.337313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076700.657181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076700.849169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076701.361215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076701.361219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076701.361243:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076701.361278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076701.681149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076701.873234:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076702.385127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076702.385134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076702.385167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076702.385210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076702.705179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076702.897126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076703.409172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076703.409172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076703.409177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076703.409212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076703.729152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076703.729227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076703.729236:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076703.729241:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076703.729254:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2409 00000100:00080000:3.0:1769076703.729271:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076703.729282:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076703.729342:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913337984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076703.729375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076703.729387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076703.729390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076703.729395:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2409 00000100:00080000:3.0:1769076703.729400:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076703.729405:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076703.729416:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076703.729420:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913337984:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076703.729433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913338112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076703.729446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076703.729450:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076703.729452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076703.729456:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2409 00000100:00080000:3.0:1769076703.729460:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076703.729481:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076703.729486:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076703.729507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913338240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076703.729515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076703.729518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076703.729520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076703.729523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2409 00000100:00080000:3.0:1769076703.729527:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076703.729533:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076703.729535:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076703.729539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076703.729550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913338368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076703.729558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076703.729561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076703.729563:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076703.729577:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2409 00000100:00080000:3.0:1769076703.729579:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076703.729582:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076703.729586:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076703.729594:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076703/real 1769076703] req@ffff935002a7f100 x1855008913337984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076703.729600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913338496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076703.729622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076703.729626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076703.729629:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076703.729630:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913337984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076703.729637:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2409 00000100:00080000:3.0:1769076703.729643:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076703.729650:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076703.729652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076703.729654:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076703.729660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076703.729668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913337984:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076703.729669:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913338624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076703.729686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076703.729689:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076703.729692:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076703.729697:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2409 00000100:00100000:0.0:1769076703.729700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913338112:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076703.729704:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076703.729713:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076703.729722:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076703.729749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076703.729761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913338752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076703.729763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076703/real 1769076703] req@ffff935002a7c380 x1855008913338112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076703.729802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913338112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076703.729811:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076703.729814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076703.729819:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913338112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076703.729825:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913338240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076703.729862:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076703.729872:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076703/real 1769076703] req@ffff935002a7ca80 x1855008913338240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076703.729889:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913338240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076703.729902:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076703.729907:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076703.729937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913338240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076703.729945:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913338368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076703.729970:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076703.730007:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076703/real 1769076703] req@ffff935002a7e300 x1855008913338368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076703.730019:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913338368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076703.730028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076703.730031:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076703.730034:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913338368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076703.730040:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913338496:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076703.730076:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076703.730159:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076703/real 1769076703] req@ffff935002a7f800 x1855008913338496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076703.730175:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913338496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076703.730183:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076703.730185:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076703.730188:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913338496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076703.730193:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913338624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076703.730209:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076703.730217:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076703/real 1769076703] req@ffff935002a7fb80 x1855008913338624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076703.730245:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913338624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076703.730252:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076703.730255:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076703.730258:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913338624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076703.730279:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913338752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076703.730294:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076703.730303:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076703/real 1769076703] req@ffff935002a7c700 x1855008913338752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076703.730313:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913338752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076703.730336:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076703.730339:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076703.730343:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913338752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076703.921212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076704.433269:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076704.433301:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076704.433306:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076704.433419:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076704.753198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076704.945190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076705.457150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076705.457151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076705.457218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076705.457223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076705.777218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076705.969226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076706.481168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076706.481169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076706.481169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076706.481234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076706.801193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076706.993128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076707.505126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076707.505130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076707.505130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076707.505166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076707.825136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076708.017130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076708.529102:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076708.529123:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076708.529131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076708.529207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076708.849248:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076708.849256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076708.849267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076708.849274:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076708.849286:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2414 00000100:00080000:2.0:1769076708.849302:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076708.849315:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076708.849375:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913338880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076708.849428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076708.849435:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076708.849438:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076708.849445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2414 00000100:00080000:2.0:1769076708.849452:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076708.849459:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076708.849473:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076708.849484:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913338880:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076708.849520:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913339008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076708.849536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076708.849540:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076708.849542:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076708.849547:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2414 00000100:00080000:2.0:1769076708.849551:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076708.849556:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076708.849562:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076708.849576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913339136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076708.849594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076708.849598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076708.849600:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076708.849609:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2414 00000800:00000100:0.0:1769076708.849612:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076708.849616:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076708.849622:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076708.849627:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076708.849640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913339264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076708.849650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076708.849654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076708.849656:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076708.849662:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2414 00000100:00080000:2.0:1769076708.849667:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076708.849675:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076708/real 1769076708] req@ffff935002a61880 x1855008913338880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076708.849687:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076708.849696:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076708.849703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913338880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076708.849714:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913339392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076708.849723:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076708.849734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076708.849739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076708.849741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076708.849748:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076708.849753:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913338880:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076708.849758:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2414 00000100:00080000:2.0:1769076708.849767:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076708.849772:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076708.849777:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076708.849789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913339520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076708.849795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339008:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076708.849816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076708.849838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076708.849841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076708.849847:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2414 00000100:00080000:2.0:1769076708.849851:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076708.849855:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076708.849861:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076708.849876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913339648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076708.849887:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076708.849928:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076708/real 1769076708] req@ffff935002a62a00 x1855008913339008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076708.849946:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913339008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076708.849957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076708.849961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076708.849966:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076708.849975:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076708.850078:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076708.850115:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076708/real 1769076708] req@ffff935002a60000 x1855008913339136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076708.850131:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913339136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076708.850142:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076708.850146:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076708.850150:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076708.850157:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076708.850183:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076708.850195:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076708/real 1769076708] req@ffff935002a60a80 x1855008913339264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076708.850210:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913339264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076708.850220:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076708.850223:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076708.850227:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076708.850233:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913339392:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076708.850252:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076708.850262:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076708/real 1769076708] req@ffff935002a61500 x1855008913339392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076708.850276:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913339392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076708.850286:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076708.850289:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076708.850293:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913339392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076708.850317:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913339520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076708.850335:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076708.850347:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076708/real 1769076708] req@ffff935002a62300 x1855008913339520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076708.850361:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913339520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076708.850372:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076708.850376:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076708.850380:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913339520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076708.850386:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913339648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076708.850403:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076708.850414:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076708/real 1769076708] req@ffff935002a62680 x1855008913339648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076708.850427:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913339648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076708.850437:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076708.850441:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076708.850444:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913339648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076709.041235:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076709.553168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076709.553184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076709.553212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076709.553301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076709.873233:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076710.065190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076710.577246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076710.577247:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076710.577248:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076710.577335:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076710.897155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076711.089134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076711.601155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076711.601159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076711.601169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076711.601357:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076711.921168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076712.113221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076712.625188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076712.625237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076712.625237:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076712.625349:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076712.945123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076713.137160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076713.649142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076713.649142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076713.649160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076713.649211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076713.969171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076713.969242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076713.969272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076713.969278:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076713.969289:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2419 00000100:00080000:2.0:1769076713.969304:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076713.969328:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076713.969374:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913339776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076713.969410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076713.969414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076713.969416:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076713.969421:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2419 00000100:00080000:2.0:1769076713.969426:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076713.969432:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076713.969444:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076713.969459:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913339776:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076713.969464:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913339904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076713.969487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076713.969491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076713.969493:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076713.969499:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2419 00000100:00080000:2.0:1769076713.969502:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076713.969505:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076713.969510:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076713.969523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913340032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076713.969534:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076713.969550:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076713.969552:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076713.969556:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2419 00000100:00080000:2.0:1769076713.969561:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076713.969565:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076713.969569:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076713.969575:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076713.969585:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913340160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076713.969597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076713.969600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076713.969602:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076713.969616:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2419 00000100:00080000:2.0:1769076713.969623:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076713.969626:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076713.969630:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076713.969645:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913340288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076713.969657:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff935002a61180 x1855008913339776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076713.969659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076713.969675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076713.969677:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076713.969682:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2419 00000100:00080000:2.0:1769076713.969694:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076713.969707:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076713.969713:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076713.969719:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913339776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076713.969728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913340416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076713.969745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076713.969766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076713.969783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076713.969784:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913339776:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076713.969792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076713.969800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076713.969807:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2419 00000100:00080000:2.0:1769076713.969812:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076713.969816:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076713.969820:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076713.969834:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913340544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076713.969838:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339904:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076713.969887:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076713.969904:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff935002a61f80 x1855008913339904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076713.969933:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913339904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076713.969942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076713.969945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076713.969948:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913339904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076713.969954:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076713.969975:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076713.969984:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff935002a61c00 x1855008913340032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076713.969993:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913340032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076713.969999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076713.970003:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076713.970007:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076713.970014:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076713.970036:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076713.970077:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff935002a63100 x1855008913340160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076713.970093:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913340160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076713.970100:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076713.970103:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076713.970105:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076713.970111:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913340288:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076713.970128:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076713.970138:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff935002a62680 x1855008913340288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076713.970152:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913340288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076713.970157:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076713.970160:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076713.970162:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913340288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076713.970166:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913340416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076713.970178:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076713.970196:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff935002a62300 x1855008913340416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076713.970205:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913340416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076713.970215:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076713.970219:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076713.970223:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913340416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076713.970249:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913340544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076713.970284:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076713.970296:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff935002a61500 x1855008913340544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076713.970305:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913340544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076713.970326:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076713.970329:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076713.970332:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913340544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076714.161169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076714.673122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076714.673141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076714.673180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076714.673195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076714.993137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076715.185109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076715.697125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076715.697126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076715.697126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076715.697193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076716.017203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076716.209142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076716.721225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076716.721225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076716.721236:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076716.721321:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076717.041168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076717.233133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076717.745197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076717.745232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076717.745247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076717.745326:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076718.065177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076718.257171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076718.769283:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076718.769284:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076718.769286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076718.769290:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076719.089228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076719.089361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076719.089376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076719.089384:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076719.089400:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2424 00000100:00080000:2.0:1769076719.089441:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076719.089451:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076719.089525:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913340672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076719.089565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076719.089571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076719.089575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076719.089584:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2424 00000100:00080000:2.0:1769076719.089592:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076719.089599:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076719.089612:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076719.089634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913340672:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076719.089636:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913340800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076719.089674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076719.089678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076719.089681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076719.089691:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2424 00000100:00080000:2.0:1769076719.089699:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076719.089706:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076719.089712:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076719.089745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913340928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076719.089759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076719.089765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076719.089767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076719.089771:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2424 00000100:00080000:2.0:1769076719.089778:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076719.089783:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076719.089788:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076719.089800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913341056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076719.089810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076719.089815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076719.089818:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076719.089846:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2424 00000100:00080000:2.0:1769076719.089853:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076719.089879:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076719.089890:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076719.089905:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076719.089917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913341184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076719.089936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076719.089962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076719.089965:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076719.089973:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2424 00000100:00080000:2.0:1769076719.089986:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076719.089996:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076719.090006:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340928:192.168.204.121@tcp:8:kworker.0 00000008:00100000:2.0:1769076719.090011:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076719.090068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913341312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076719.090090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076719.090138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076719.090141:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076719.090150:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2424 00000100:00100000:0.0:1769076719.090168:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341056:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076719.090189:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076719.090199:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076719.090215:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076719.090268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913341440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076719.090270:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913341184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076719.090348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913341312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076719.090390:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913341440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076719.281221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076719.793144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076719.793153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076719.793192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076719.793255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076720.113196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076720.305157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076720.817198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076720.817258:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076720.817259:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076720.817328:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076721.137220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076721.329196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076721.841243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076721.841245:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076721.841245:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076721.841278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076722.161217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076722.161530:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076722.161596:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076722.161626:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076722.161683:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076722.161699:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076722.161712:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076722.161727:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076722.161763:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076722.161827:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076722.161836:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076719/real 1769076722] req@ffff935002a61500 x1855008913341184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076722.161898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913341184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076722.161921:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076722.161932:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076722.161942:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913341184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076722.161958:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076719/real 1769076722] req@ffff935002a62300 x1855008913341312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076722.161984:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913341312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076722.162000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076722.162019:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076722.162027:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913341312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076722.162041:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076719/real 1769076722] req@ffff935002a60000 x1855008913340800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076722.162112:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913340800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076722.162131:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076722.162140:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076722.162147:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076722.162162:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076719/real 1769076722] req@ffff935002a62a00 x1855008913340928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076722.162186:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913340928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076722.162202:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076722.162210:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076722.162216:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913340928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076722.162231:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076719/real 1769076722] req@ffff935002a61880 x1855008913341056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076722.162256:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913341056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076722.162273:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076722.162281:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076722.162287:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076722.162301:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076719/real 1769076722] req@ffff935002a60a80 x1855008913340672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076722.162330:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913340672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076722.162346:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076722.162353:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076722.162359:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913340672:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076722.162443:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076719/real 1769076722] req@ffff935002a62680 x1855008913341440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076722.162467:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913341440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076722.162482:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076722.162489:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076722.162495:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913341440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076722.353106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076722.865176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076722.865190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076722.865254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076722.865259:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076723.185317:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076723.377221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076723.889222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076723.889268:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076723.889272:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076723.889293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076724.209268:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076724.209291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076724.209298:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076724.209305:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076724.209313:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2429 00000100:00080000:1.0:1769076724.209349:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076724.209362:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076724.209428:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913341568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076724.209468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076724.209473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076724.209476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076724.209482:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2429 00000100:00080000:1.0:1769076724.209486:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076724.209491:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076724.209507:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076724.209535:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913341696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076724.209553:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913341568:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076724.209569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076724.209581:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076724.209587:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076724.209603:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2429 00000100:00080000:1.0:1769076724.209613:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076724.209624:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076724.209638:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076724.209669:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076724.209687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913341824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076724.209732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076724.209774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076724/real 1769076724] req@ffff9350100c8380 x1855008913341568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076724.209797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076724.209807:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076724.209823:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2429 00000100:00080000:1.0:1769076724.209834:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076724.209838:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913341568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076724.209853:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076724.209881:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1769076724.209890:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076724.209907:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769076724.209918:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913341952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076724.209929:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913341568:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076724.209965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076724.209979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076724.209985:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076724.209997:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2429 00000100:00080000:1.0:1769076724.210006:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076724.210015:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076724.210025:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341696:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769076724.210041:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076724.210133:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076724.210170:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076724/real 1769076724] req@ffff9350100ca300 x1855008913341696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076724.210191:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913342080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076724.210209:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913341696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076724.210251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076724.210253:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076724.210273:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076724.210275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076724.210286:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076724.210291:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341696:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076724.210311:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2429 00000100:00080000:1.0:1769076724.210327:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076724.210328:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341824:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769076724.210351:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076724.210369:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076724.210396:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076724.210407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913342208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076724.210437:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076724/real 1769076724] req@ffff9350100ca680 x1855008913341824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076724.210440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076724.210465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076724.210477:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076724.210497:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2429 00000100:00080000:1.0:1769076724.210515:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076724.210529:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076724.210545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913341824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076724.210552:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076724.210589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076724.210604:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769076724.210605:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913342336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076724.210616:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076724.210651:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076724.210729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076724.210774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076724/real 1769076724] req@ffff9350100c8e00 x1855008913341952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076724.210799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913341952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076724.210816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076724.210823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076724.210830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913341952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076724.210878:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342080:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076724.210926:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076724.210948:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076724/real 1769076724] req@ffff9350100c9880 x1855008913342080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076724.210971:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913342080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076724.210993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076724.210998:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076724.211004:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076724.211015:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076724.211090:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076724.211113:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076724/real 1769076724] req@ffff9350100c9500 x1855008913342208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076724.211134:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913342208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076724.211148:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076724.211153:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076724.211159:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076724.211170:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076724.211204:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076724.211224:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076724/real 1769076724] req@ffff9350100caa00 x1855008913342336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076724.211246:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913342336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076724.211260:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076724.211265:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076724.211271:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076724.401154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076724.913199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076724.913275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076724.913304:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076724.913334:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076725.233306:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076725.425181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076725.937227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076725.937239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076725.937240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076725.937262:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076726.257175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076726.449117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076726.961161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076726.961166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076726.961167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076726.961303:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076727.281203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076727.473180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076727.985162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076727.985201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076727.985214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076727.985227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076728.305179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076728.497145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076729.009211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076729.009247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076729.009250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076729.009293:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076729.329166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076729.329228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076729.329241:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076729.329248:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076729.329266:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2435 00000100:00080000:3.0:1769076729.329284:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076729.329296:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076729.329361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913342464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076729.329400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076729.329404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076729.329407:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076729.329412:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2435 00000100:00080000:3.0:1769076729.329417:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076729.329422:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076729.329436:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076729.329453:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913342592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076729.329467:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913342464:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076729.329511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076729.329516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076729.329520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076729.329530:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2435 00000100:00080000:3.0:1769076729.329537:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076729.329544:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076729.329552:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076729.329576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913342720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076729.329593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076729.329600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076729.329603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076729.329610:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2435 00000100:00080000:3.0:1769076729.329617:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076729.329627:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076729.329633:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076729.329640:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076729.329709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913342848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076729.329726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076729.329732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076729.329734:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076729.329741:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2435 00000100:00080000:3.0:1769076729.329746:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076729.329751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076729.329757:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076729.329771:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913342976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076729.329780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076729.329784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076729.329786:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076729.329790:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2435 00000100:00080000:3.0:1769076729.329792:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076729.329795:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076729.329799:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076729.329808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913343104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076729.329816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076729.329820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076729.329822:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076729.329825:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2435 00000100:00080000:3.0:1769076729.329827:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076729.329830:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076729.329834:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076729.329860:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913343232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076729.330008:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076729/real 1769076729] req@ffff935002a7c000 x1855008913342464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076729.330037:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913342464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076729.330081:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076729.330099:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076729.330105:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913342464:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076729.330141:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913342592:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076729.330181:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076729.330199:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076729/real 1769076729] req@ffff935002a7df80 x1855008913342592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076729.330216:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913342592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076729.330227:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076729.330232:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076729.330236:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913342592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076729.330245:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913342720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076729.330266:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076729.330278:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076729/real 1769076729] req@ffff935002a7d180 x1855008913342720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076729.330293:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913342720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076729.330303:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076729.330307:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076729.330311:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913342720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076729.330319:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913342848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076729.330337:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076729.330348:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076729/real 1769076729] req@ffff935002a7dc00 x1855008913342848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076729.330372:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913342848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076729.330382:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076729.330385:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076729.330389:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913342848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076729.330396:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342976:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076729.330414:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076729.330426:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076729/real 1769076729] req@ffff935002a7c700 x1855008913342976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076729.330440:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913342976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076729.330451:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076729.330454:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076729.330458:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913342976:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076729.330465:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913343104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076729.330497:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076729.330519:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076729/real 1769076729] req@ffff935002a7fb80 x1855008913343104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076729.330543:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913343104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076729.330569:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076729.330576:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076729.330584:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913343104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076729.330597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913343232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076729.330634:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076729.330689:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076729/real 1769076729] req@ffff935002a7f800 x1855008913343232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076729.330715:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913343232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076729.330733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076729.330739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076729.330745:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913343232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076729.521193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076730.033214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076730.033218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076730.033297:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076730.033297:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076730.353147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076730.545127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076731.057127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076731.057172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076731.057174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076731.057258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076731.377212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076731.569221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076732.081187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076732.081284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076732.081380:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076732.081417:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076732.401111:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076732.593118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076733.105130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076733.105131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076733.105148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076733.105195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076733.425110:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076733.617148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076734.129139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076734.129139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076734.129139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076734.129167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076734.449113:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076734.449161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076734.449170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076734.449174:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076734.449183:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2440 00000100:00080000:2.0:1769076734.449194:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076734.449200:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076734.449246:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913343360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076734.449273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076734.449275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076734.449277:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076734.449280:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2440 00000100:00080000:2.0:1769076734.449283:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076734.449286:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076734.449294:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076734.449310:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913343488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076734.449312:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913343360:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076734.449334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076734.449336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076734.449338:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076734.449344:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2440 00000100:00080000:2.0:1769076734.449347:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076734.449350:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076734.449354:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076734.449364:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913343616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076734.449372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076734.449374:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076734.449375:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076734.449378:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2440 00000100:00080000:2.0:1769076734.449380:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076734.449383:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076734.449387:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076734.449394:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913343744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076734.449402:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076734.449414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076734.449417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076734.449419:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076734.449426:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2440 00000100:00080000:2.0:1769076734.449436:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076734.449440:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076734.449444:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076734.449455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913343872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076734.449460:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076734/real 1769076734] req@ffff935002a63100 x1855008913343360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076734.449485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076734.449490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076734.449491:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076734.449494:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2440 00000100:00080000:2.0:1769076734.449500:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076734.449507:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076734.449515:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076734.449516:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913343360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076734.449539:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076734.449545:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076734.449557:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913343360:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076734.449559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913344000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076734.449597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076734.449598:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913343488:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076734.449607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076734.449611:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076734.449615:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2440 00000100:00080000:2.0:1769076734.449619:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076734.449623:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076734.449628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769076734.449629:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076734.449648:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913344128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076734.449653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076734/real 1769076734] req@ffff935002a61c00 x1855008913343488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076734.449677:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913343488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076734.449685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076734.449689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076734.449694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913343488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076734.449701:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913343616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076734.449714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076734.449721:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076734/real 1769076734] req@ffff935002a61f80 x1855008913343616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076734.449727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913343616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076734.449733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076734.449735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076734.449737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913343616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076734.449741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913343744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076734.449750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076734.449756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076734/real 1769076734] req@ffff935002a61180 x1855008913343744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076734.449763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913343744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076734.449768:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076734.449770:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076734.449772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913343744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076734.449775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913343872:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076734.449785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076734.449791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076734/real 1769076734] req@ffff935002a62680 x1855008913343872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076734.449803:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913343872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076734.449809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076734.449810:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076734.449812:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913343872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076734.449825:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076734.449835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076734.449841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076734/real 1769076734] req@ffff935002a60a80 x1855008913344000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076734.449849:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913344000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076734.449854:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076734.449855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076734.449857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076734.449861:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076734.449870:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076734.449876:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076734/real 1769076734] req@ffff935002a61880 x1855008913344128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076734.449882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913344128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076734.449892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076734.449894:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076734.449896:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076734.641142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076735.153126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076735.153154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076735.153172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076735.153212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076735.473244:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076735.665126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076736.177171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076736.177202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076736.177202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076736.177243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076736.497176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076736.689182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076737.201128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076737.201129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076737.201135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076737.201254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076737.521109:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076737.713164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076738.225142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076738.225142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076738.225148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076738.225186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076738.545164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076738.737137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076739.249169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076739.249175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076739.249188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076739.249257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076739.569193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076739.569243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076739.569250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076739.569254:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076739.569263:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2445 00000100:00080000:3.0:1769076739.569274:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076739.569280:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076739.569319:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913344256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076739.569343:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076739.569346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076739.569347:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076739.569350:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2445 00000100:00080000:3.0:1769076739.569352:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076739.569354:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076739.569363:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076739.569374:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913344384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076739.569381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076739.569384:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076739.569385:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076739.569388:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2445 00000100:00080000:3.0:1769076739.569389:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076739.569393:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076739.569397:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913344256:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076739.569398:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076739.569407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913344512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076739.569414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076739.569417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076739.569418:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076739.569421:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2445 00000100:00080000:3.0:1769076739.569423:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076739.569425:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076739.569428:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076739.569435:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913344640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076739.569441:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076739.569444:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076739.569445:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076739.569449:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2445 00000100:00080000:3.0:1769076739.569451:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076739.569454:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076739.569457:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076739.569464:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913344768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076739.569470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076739.569472:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076739.569473:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076739.569475:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2445 00000100:00080000:3.0:1769076739.569478:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076739.569496:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076739.569507:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076739.569512:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076739.569521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913344896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076739.569528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076739.569530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076739.569531:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076739.569534:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2445 00000100:00080000:3.0:1769076739.569537:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076739.569540:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076739.569543:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076739.569554:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913345024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076739.569556:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076739/real 1769076739] req@ffff935002a7e300 x1855008913344256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076739.569586:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913344256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076739.569597:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076739.569602:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076739.569608:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913344256:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076739.569616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913344384:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076739.569642:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076739.569655:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076739/real 1769076739] req@ffff935002a7ca80 x1855008913344384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076739.569669:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913344384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076739.569679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076739.569683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076739.569686:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913344384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076739.569711:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913344512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076739.569732:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076739.569744:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076739/real 1769076739] req@ffff935002a7c380 x1855008913344512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076739.569757:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913344512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076739.569766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076739.569769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076739.569773:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913344512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076739.569779:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913344640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076739.569796:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076739.569806:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076739/real 1769076739] req@ffff935002a7f100 x1855008913344640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076739.569819:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913344640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076739.569828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076739.569831:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076739.569835:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913344640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076739.569840:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344768:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076739.569854:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076739.569863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076739/real 1769076739] req@ffff935002a7f800 x1855008913344768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076739.569887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913344768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076739.569896:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076739.569899:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076739.569903:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076739.569908:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076739.569924:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076739.569934:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076739/real 1769076739] req@ffff935002a7fb80 x1855008913344896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076739.569945:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913344896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076739.569954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076739.569965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076739.569969:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913344896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076739.569974:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076739.569991:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076739.570000:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076739/real 1769076739] req@ffff935002a7c700 x1855008913345024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076739.570012:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913345024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076739.570021:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076739.570024:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076739.570027:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076739.761136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076740.273181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076740.273181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076740.273312:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076740.273406:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076740.593224:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076740.785188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076741.297146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076741.297150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076741.297235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076741.297322:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076741.617127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076741.809133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076742.321197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076742.321197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076742.321197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076742.321246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076742.641154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076742.833175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076743.345130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076743.345140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076743.345175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076743.345192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076743.665146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076743.857114:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076744.369177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076744.369185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076744.369197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076744.369235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076744.689166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076744.689177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076744.689190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076744.689195:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076744.689207:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2450 00000100:00080000:2.0:1769076744.689222:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076744.689230:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076744.689274:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913345152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076744.689300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076744.689303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076744.689305:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076744.689309:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2450 00000100:00080000:2.0:1769076744.689313:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076744.689318:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076744.689327:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076744.689338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913345152:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076744.689342:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913345280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076744.689359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076744.689362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076744.689364:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076744.689370:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2450 00000100:00080000:2.0:1769076744.689380:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076744.689383:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076744.689389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076744.689416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913345408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076744.689426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076744.689430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076744.689432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076744.689438:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076744.689442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2450 00000100:00080000:2.0:1769076744.689452:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076744.689458:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076744.689467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076744.689484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913345536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076744.689495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076744.689498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769076744.689502:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076744/real 1769076744] req@ffff935002a62a00 x1855008913345152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076744.689503:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076744.689524:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2450 00000100:00080000:2.0:1769076744.689532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076744.689534:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913345152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076744.689547:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076744.689550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076744.689557:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769076744.689557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076744.689563:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913345152:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076744.689582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913345664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076744.689596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076744.689597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913345280:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076744.689612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076744.689615:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076744.689620:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2450 00000100:00080000:2.0:1769076744.689628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076744.689633:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076744.689640:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769076744.689644:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076744.689653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076744/real 1769076744] req@ffff935002a60000 x1855008913345280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076744.689665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913345792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076744.689666:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913345280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076744.689697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076744.689705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076744.689708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913345280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076744.689721:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913345408:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076744.689721:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076744.689738:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076744.689741:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076744.689746:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2450 00000100:00080000:2.0:1769076744.689755:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076744.689756:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076744.689767:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076744.689770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076744/real 1769076744] req@ffff935002a62300 x1855008913345408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076744.689786:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076744.689788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913345408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076744.689802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076744.689806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076744.689811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913345408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076744.689813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913345920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076744.689826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913345536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076744.689853:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076744.689861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076744/real 1769076744] req@ffff935002a61500 x1855008913345536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076744.689871:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913345536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076744.689878:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076744.689882:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076744.689884:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913345536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076744.689897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345664:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076744.689907:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076744.689913:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076744/real 1769076744] req@ffff935002a61880 x1855008913345664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076744.689920:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913345664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076744.689926:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076744.689928:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076744.689930:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076744.689933:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076744.689944:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076744.689949:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076744/real 1769076744] req@ffff935002a60a80 x1855008913345792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076744.689957:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913345792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076744.689962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076744.689963:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076744.689965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076744.689969:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076744.689978:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076744.689983:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076744/real 1769076744] req@ffff935002a62680 x1855008913345920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076744.689989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913345920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076744.689998:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076744.690000:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076744.690002:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913345920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076744.881173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076745.393177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076745.393181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076745.393186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076745.393283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076745.713180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076745.905135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076746.417134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076746.417135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076746.417142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076746.417246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076746.737156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076746.929147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076747.441139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076747.441139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076747.441145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076747.441255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076747.761118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076747.953126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076748.465126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076748.465137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076748.465144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076748.465162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076748.785153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076748.977181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076749.489103:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076749.489132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076749.489168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076749.489220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076749.809173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076749.809185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076749.809194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076749.809199:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076749.809211:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2455 00000100:00080000:1.0:1769076749.809224:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076749.809232:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076749.809276:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913346048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076749.809305:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076749.809324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076749.809326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076749.809331:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2455 00000100:00080000:1.0:1769076749.809335:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076749.809341:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076749.809353:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076749.809362:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913346048:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076749.809372:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913346176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076749.809394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076749.809402:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076749.809404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076749.809409:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2455 00000100:00080000:1.0:1769076749.809420:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076749.809424:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076749.809430:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076749.809436:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076749.809444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913346304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076749.809455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076749.809458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076749.809461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076749.809465:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2455 00000100:00080000:1.0:1769076749.809469:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076749.809473:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076749.809479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076749.809487:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076749/real 1769076749] req@ffff9350100cb800 x1855008913346048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076749.809493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913346432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076749.809516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076749.809521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769076749.809523:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913346048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076749.809525:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076749.809535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2455 00000100:00080000:1.0:1769076749.809543:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076749.809546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076749.809547:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076749.809556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076749.809557:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076749.809564:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913346048:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076749.809569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913346560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076749.809580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076749.809583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076749.809585:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076749.809589:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2455 00000100:00080000:1.0:1769076749.809592:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076749.809596:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913346176:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076749.809599:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076749.809609:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076749.809623:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913346688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076749.809632:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076749.809642:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076749.809646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076749.809649:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076749.809654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076749/real 1769076749] req@ffff9350100caa00 x1855008913346176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076749.809656:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2455 00000100:00080000:1.0:1769076749.809664:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076749.809672:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076749.809679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913346176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076749.809679:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076749.809696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076749.809698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913346816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076749.809705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076749.809714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913346176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076749.809724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913346304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076749.809745:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076749.809755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076749/real 1769076749] req@ffff9350100c9500 x1855008913346304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076749.809767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913346304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076749.809776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076749.809779:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076749.809783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913346304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076749.809789:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913346432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076749.809804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076749.809814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076749/real 1769076749] req@ffff9350100c9880 x1855008913346432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076749.809826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913346432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076749.809835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076749.809839:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076749.809842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913346432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076749.809858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913346560:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076749.809879:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076749.809890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076749/real 1769076749] req@ffff9350100c8e00 x1855008913346560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076749.809902:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913346560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076749.809911:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076749.809914:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076749.809918:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913346560:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076749.809936:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913346688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076749.809954:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076749.809964:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076749/real 1769076749] req@ffff9350100ca680 x1855008913346688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076749.809997:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913346688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076749.810006:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076749.810009:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076749.810013:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913346688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076749.810018:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913346816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076749.810035:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076749.810044:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076749/real 1769076749] req@ffff9350100ca300 x1855008913346816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076749.810081:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913346816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076749.810089:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076749.810092:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076749.810096:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913346816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076750.001227:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076750.513134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076750.513177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076750.513203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076750.513225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076750.833183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076751.025197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076751.537142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076751.537142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076751.537142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076751.537178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076751.857125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076752.049178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076752.561118:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076752.561130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076752.561156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076752.561158:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076752.881184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076753.073186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076753.585133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076753.585133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076753.585142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076753.585188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076753.905168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076754.097127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076754.609125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076754.609125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076754.609152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076754.609184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076754.929152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076754.929152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076754.929161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076754.929166:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076754.929175:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2460 00000100:00080000:1.0:1769076754.929187:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076754.929193:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076754.929253:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913346944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076754.929293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076754.929297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076754.929300:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076754.929305:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2460 00000100:00080000:1.0:1769076754.929309:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076754.929313:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076754.929323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076754.929336:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913346944:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076754.929337:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913347072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076754.929356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076754.929359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076754.929360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076754.929362:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2460 00000100:00080000:1.0:1769076754.929365:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076754.929367:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076754.929371:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076754.929381:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913347200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076754.929387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076754.929390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076754.929391:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076754.929393:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2460 00000100:00080000:1.0:1769076754.929396:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076754.929399:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076754.929402:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076754.929409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913347328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076754.929415:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076754.929422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076754.929425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076754.929427:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076754.929430:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2460 00000100:00080000:1.0:1769076754.929434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076754.929437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076754.929440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076754.929448:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913347456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076754.929454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076754.929457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076754.929461:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9350100c8380 x1855008913346944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076754.929461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076754.929469:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2460 00000100:00080000:1.0:1769076754.929474:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076754.929479:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076754.929483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076754.929489:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913346944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076754.929494:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913347584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076754.929507:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076754.929508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076754.929514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076754.929517:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076754.929518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076754.929527:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913346944:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076754.929528:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2460 00000100:00080000:1.0:1769076754.929532:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076754.929537:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076754.929540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076754.929546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913347712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076754.929554:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347072:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076754.929580:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076754.929597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9350100c9f80 x1855008913347072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.929610:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913347072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.929621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.929625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076754.929629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347072:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076754.929636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076754.929657:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076754.929666:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9350100cad80 x1855008913347200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.929674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913347200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.929680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.929683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076754.929685:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076754.929689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076754.929703:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076754.929713:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9350100c8000 x1855008913347328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.929726:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913347328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.929735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.929739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076754.929743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076754.929750:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913347456:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076754.929769:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076754.929779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9350100cb100 x1855008913347456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.929787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913347456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.929794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.929796:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076754.929798:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913347456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076754.929802:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913347584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076754.929814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076754.929820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9350100c9180 x1855008913347584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.929828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913347584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.929833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.929835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076754.929838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913347584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076754.929852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913347712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076754.929862:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076754.929868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9350100c9c00 x1855008913347712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.929875:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913347712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.929881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.929883:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076754.929885:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913347712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076755.121144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076755.633143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076755.633144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076755.633183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076755.633307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076755.953176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076756.145113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076756.657131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076756.657131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076756.657180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076756.657266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076756.977136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076757.169127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076757.681170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076757.681171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076757.681171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076757.681284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076758.001107:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076758.193106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076758.705138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076758.705138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076758.705255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076758.705284:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076759.025150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076759.217124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076759.729153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076759.729159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076759.729173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076759.729238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076760.049104:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076760.049161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076760.049168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076760.049172:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076760.049180:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2465 00000100:00080000:0.0:1769076760.049192:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076760.049204:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076760.049242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913347840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076760.049267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076760.049269:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076760.049271:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076760.049274:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2465 00000100:00080000:0.0:1769076760.049277:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076760.049279:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076760.049289:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076760.049298:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913347968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076760.049306:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076760.049309:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076760.049310:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076760.049312:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2465 00000100:00080000:0.0:1769076760.049315:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076760.049317:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076760.049320:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076760.049323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913347840:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076760.049330:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913348096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076760.049339:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076760.049341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076760.049342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076760.049345:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2465 00000100:00080000:0.0:1769076760.049347:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076760.049349:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076760.049352:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076760.049361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913348224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076760.049368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076760.049371:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076760.049373:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076760.049377:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2465 00000100:00080000:0.0:1769076760.049381:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076760.049386:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076760.049391:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076760.049401:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076760.049406:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913348352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076760.049418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076760.049423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076760.049426:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076760.049430:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2465 00000100:00080000:0.0:1769076760.049434:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076760.049438:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076760.049444:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076760.049448:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076760/real 1769076760] req@ffff9350103f9500 x1855008913347840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076760.049458:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913348480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076760.049470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076760.049473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076760.049476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076760.049478:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913347840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076760.049482:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2465 00000100:00080000:0.0:1769076760.049488:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076760.049494:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076760.049501:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076760.049504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076760.049513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076760.049514:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913348608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076760.049522:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913347840:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076760.049535:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347968:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076760.049557:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076760.049569:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076760/real 1769076760] req@ffff9350103f9f80 x1855008913347968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076760.049582:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913347968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076760.049592:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076760.049595:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076760.049599:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913347968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076760.049625:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076760.049641:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076760.049652:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076760/real 1769076760] req@ffff9350103fb800 x1855008913348096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076760.049664:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913348096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076760.049673:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076760.049677:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076760.049681:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076760.049686:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076760.049701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076760.049710:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076760/real 1769076760] req@ffff9350103fad80 x1855008913348224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076760.049722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913348224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076760.049731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076760.049734:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076760.049738:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076760.049744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913348352:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076760.049759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076760.049768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076760/real 1769076760] req@ffff9350103fbb80 x1855008913348352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076760.049779:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913348352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076760.049788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076760.049791:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076760.049795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913348352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076760.049800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913348480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076760.049818:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076760.049828:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076760/real 1769076760] req@ffff9350103fb480 x1855008913348480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076760.049848:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913348480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076760.049857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076760.049861:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076760.049864:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913348480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076760.049869:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913348608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076760.049884:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076760.049893:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076760/real 1769076760] req@ffff9350103f9c00 x1855008913348608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076760.049905:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913348608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076760.049915:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076760.049918:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076760.049921:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913348608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076760.241148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076760.753112:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076760.753118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076760.753164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076760.753178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076761.073189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076761.265135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076761.777141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076761.777143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076761.777152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076761.777198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076762.097205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076762.289189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076762.801134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076762.801134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076762.801144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076762.801177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076763.121149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076763.313143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076763.825147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076763.825151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076763.825187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076763.825191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076764.145129:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076764.337143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076764.849189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076764.849189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076764.849227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076764.849230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:0.0:1769076765.169192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076765.169210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1769076765.169212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076765.169217:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076765.169233:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2470 00000100:00080000:0.0:1769076765.169250:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076765.169258:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076765.169308:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913348736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076765.169358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076765.169364:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076765.169367:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076765.169372:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2470 00000100:00080000:0.0:1769076765.169377:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076765.169382:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076765.169408:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076765.169423:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913348864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076765.169424:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913348736:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076765.169439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076765.169442:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076765.169443:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076765.169447:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2470 00000100:00080000:0.0:1769076765.169450:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076765.169452:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076765.169456:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076765.169465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913348992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076765.169472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076765.169474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076765.169476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076765.169478:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2470 00000100:00080000:0.0:1769076765.169481:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076765.169484:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076765.169487:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076765.169505:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913349120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076765.169511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076765.169516:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076765.169517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076765.169521:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076765.169525:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2470 00000100:00080000:0.0:1769076765.169528:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076765.169530:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076765.169534:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076765.169543:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913349248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076765.169549:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076765.169552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076765.169554:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076765.169559:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2470 00000100:00100000:2.0:1769076765.169560:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076765/real 1769076765] req@ffff9350103f9180 x1855008913348736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076765.169562:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076765.169567:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076765.169573:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076765.169587:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913349376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076765.169596:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913348736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076765.169609:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076765.169614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076765.169615:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076765.169617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076765.169620:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2470 00000100:00080000:0.0:1769076765.169623:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076765.169624:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076765.169628:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076765.169633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913348736:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769076765.169635:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076765.169645:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913349504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076765.169659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076765.169677:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076765.169686:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076765/real 1769076765] req@ffff9350103faa00 x1855008913348864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076765.169695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913348864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076765.169701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076765.169704:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076765.169706:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076765.169710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076765.169721:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076765.169727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076765/real 1769076765] req@ffff9350103f8700 x1855008913348992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076765.169735:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913348992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076765.169746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076765.169748:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076765.169750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913348992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076765.169754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913349120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076765.169763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076765.169768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076765/real 1769076765] req@ffff9350103f8380 x1855008913349120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076765.169775:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913349120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076765.169780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076765.169782:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076765.169784:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913349120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076765.169788:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913349248:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076765.169796:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076765.169802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076765/real 1769076765] req@ffff9350103fb100 x1855008913349248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076765.169810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913349248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076765.169818:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076765.169821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076765.169825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913349248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076765.169831:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913349376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076765.169851:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076765.169863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076765/real 1769076765] req@ffff9350103f9c00 x1855008913349376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076765.169876:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913349376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076765.169882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076765.169885:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076765.169887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913349376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076765.169892:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913349504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076765.169925:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076765.169934:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076765/real 1769076765] req@ffff9350103fb480 x1855008913349504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076765.169943:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913349504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076765.169949:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076765.169952:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076765.169955:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913349504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076765.361198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076765.873226:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076765.873232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076765.873301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076765.873310:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076766.193278:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076766.385206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076766.897120:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076766.897126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076766.897136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076766.897173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076767.217163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076767.409190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076767.921222:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076767.921223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076767.921227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076767.921264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076768.241184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076768.433210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076768.945186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076768.945207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076768.945222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076768.945282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076769.265212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076769.457159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076769.969191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076769.969220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076769.969256:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076769.969274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076770.289178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076770.289179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076770.289189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076770.289194:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076770.289203:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2475 00000100:00080000:3.0:1769076770.289215:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076770.289222:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076770.289261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913349632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076770.289286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076770.289289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076770.289291:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076770.289294:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2475 00000100:00080000:3.0:1769076770.289297:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076770.289300:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076770.289308:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076770.289319:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913349760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076770.289327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076770.289331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076770.289333:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076770.289336:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2475 00000100:00080000:3.0:1769076770.289339:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076770.289342:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076770.289345:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076770.289354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913349888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076770.289362:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913349632:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076770.289365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076770.289373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076770.289374:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076770.289378:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2475 00000100:00080000:3.0:1769076770.289381:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076770.289384:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076770.289389:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076770.289399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913350016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076770.289406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076770.289409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076770.289410:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076770.289414:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2475 00000100:00080000:3.0:1769076770.289430:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076770.289433:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076770.289438:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076770.289445:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076770.289453:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913350144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076770.289463:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076770.289466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076770.289468:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076770.289471:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2475 00000100:00080000:3.0:1769076770.289475:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076770.289478:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076770.289482:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076770.289492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913350272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076770.289499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076770.289502:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076770.289503:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076770.289506:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2475 00000100:00080000:3.0:1769076770.289508:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076770.289514:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076770.289516:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076770/real 1769076770] req@ffff935002a7dc00 x1855008913349632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076770.289519:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076770.289533:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913350400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076770.289545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913349632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076770.289558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076770.289562:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076770.289565:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913349632:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076770.289572:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913349760:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076770.289602:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076770.289618:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076770/real 1769076770] req@ffff935002a7d180 x1855008913349760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076770.289646:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913349760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076770.289656:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076770.289659:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076770.289662:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913349760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076770.289699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913349888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076770.289729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076770.289757:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076770/real 1769076770] req@ffff935002a7df80 x1855008913349888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076770.289775:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913349888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076770.289785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076770.289788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076770.289791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913349888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076770.289799:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076770.289825:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076770.289839:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076770/real 1769076770] req@ffff935002a7c000 x1855008913350016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076770.289868:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913350016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076770.289875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076770.289878:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076770.289881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076770.289886:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913350144:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076770.289905:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076770.289914:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076770/real 1769076770] req@ffff935002a7c700 x1855008913350144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076770.289929:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913350144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076770.289936:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076770.289939:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076770.289941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913350144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076770.289946:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913350272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076770.290012:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076770.290028:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076770/real 1769076770] req@ffff935002a7fb80 x1855008913350272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076770.290044:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913350272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076770.290103:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076770.290108:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076770.290112:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913350272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076770.290121:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913350400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076770.290150:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076770.290165:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076770/real 1769076770] req@ffff935002a7f800 x1855008913350400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076770.290176:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913350400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076770.290188:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076770.290191:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076770.290193:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913350400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076770.481154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076770.993135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076770.993135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076770.993136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076770.993171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076771.313128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076771.505191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076772.017179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076772.017182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076772.017215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076772.017220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076772.337180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076772.529151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076773.041164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076773.041164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076773.041164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076773.041290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076773.361129:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076773.553115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076774.065144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076774.065156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076774.065156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076774.065202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076774.385162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076774.577149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076775.089143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076775.089143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076775.089151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076775.089192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076775.409136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076775.409170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076775.409177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076775.409182:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076775.409190:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2481 00000100:00080000:1.0:1769076775.409201:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076775.409207:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076775.409243:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913350528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076775.409269:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076775.409272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076775.409273:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076775.409276:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2481 00000100:00080000:1.0:1769076775.409278:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076775.409281:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076775.409289:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076775.409299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913350656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076775.409306:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076775.409309:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076775.409310:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076775.409313:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2481 00000100:00080000:1.0:1769076775.409315:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076775.409318:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076775.409321:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076775.409329:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913350784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076775.409338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076775.409338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913350528:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076775.409344:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076775.409346:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076775.409352:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2481 00000100:00080000:1.0:1769076775.409355:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076775.409357:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076775.409360:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076775.409368:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913350912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076775.409375:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076775.409377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076775.409378:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076775.409380:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2481 00000100:00080000:1.0:1769076775.409382:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076775.409384:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076775.409387:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076775.409395:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913351040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076775.409400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076775.409403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076775.409404:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076775.409406:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2481 00000100:00080000:1.0:1769076775.409409:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076775.409411:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076775.409414:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076775.409421:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913351168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076775.409430:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076775.409436:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076775.409440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076775.409441:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076775.409444:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2481 00000100:00080000:1.0:1769076775.409446:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076775.409449:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076775.409452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076775.409479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913351296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076775.409534:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076775/real 1769076775] req@ffff9350100c8a80 x1855008913350528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076775.409557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913350528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076775.409566:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076775.409569:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076775.409572:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913350528:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076775.409578:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350656:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076775.409598:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076775.409608:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076775/real 1769076775] req@ffff9350100c9c00 x1855008913350656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076775.409616:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913350656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076775.409628:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076775.409630:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076775.409632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076775.409652:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076775.409667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076775.409678:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076775/real 1769076775] req@ffff9350100c9180 x1855008913350784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076775.409691:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913350784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076775.409704:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076775.409708:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076775.409711:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076775.409718:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076775.409739:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076775.409750:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076775/real 1769076775] req@ffff9350100cb100 x1855008913350912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076775.409763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913350912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076775.409772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076775.409775:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076775.409779:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913350912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076775.409786:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351040:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076775.409806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076775.409817:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076775/real 1769076775] req@ffff9350100c8000 x1855008913351040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076775.409842:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913351040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076775.409851:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076775.409854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076775.409858:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076775.409864:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076775.409900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076775.409912:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076775/real 1769076775] req@ffff9350100cad80 x1855008913351168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076775.409935:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913351168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076775.409945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076775.409950:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076775.409954:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076775.409961:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076775.409981:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076775.409993:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076775/real 1769076775] req@ffff9350100c9f80 x1855008913351296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076775.410006:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913351296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076775.410015:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076775.410018:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076775.410022:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076775.601153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076776.113154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076776.113156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076776.113177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076776.113306:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076776.433209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076776.625255:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076777.137263:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076777.137264:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076777.137269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076777.137373:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076777.457188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076777.649209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076778.161195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076778.161204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076778.161259:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076778.161427:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076778.481283:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076778.673141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076779.185123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076779.185161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076779.185189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076779.185212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076779.505257:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076779.697149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076780.209192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076780.209193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076780.209200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076780.209339:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076780.529177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076780.529209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076780.529217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076780.529222:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076780.529232:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2486 00000100:00080000:2.0:1769076780.529243:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076780.529249:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076780.529317:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913351424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076780.529358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076780.529363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076780.529366:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076780.529373:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2486 00000100:00080000:2.0:1769076780.529402:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076780.529406:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076780.529411:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913351424:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076780.529416:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076780.529432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913351552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076780.529441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076780.529443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076780.529445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076780.529448:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2486 00000100:00080000:2.0:1769076780.529451:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076780.529454:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076780.529457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076780.529464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913351680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076780.529489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076780.529492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076780.529493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076780.529499:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076780.529500:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2486 00000100:00080000:2.0:1769076780.529507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076780.529511:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076780.529515:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076780.529525:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913351808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076780.529532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076780.529552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076780.529553:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076780.529558:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2486 00000100:00080000:2.0:1769076780.529563:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076780.529568:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076780.529573:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076780/real 1769076780] req@ffff935002a61180 x1855008913351424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076780.529574:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076780.529593:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913351936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076780.529618:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913351424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076780.529623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076780.529634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076780.529638:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076780.529641:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076780.529643:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2486 00000100:00080000:2.0:1769076780.529650:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076780.529653:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076780.529657:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076780.529662:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913351424:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076780.529665:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076780.529677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913352064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076780.529687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076780.529690:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913351552:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076780.529692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076780.529696:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076780.529702:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2486 00000100:00080000:2.0:1769076780.529706:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076780.529709:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076780.529714:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076780.529726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076780.529726:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913352192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076780.529753:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076780/real 1769076780] req@ffff935002a61f80 x1855008913351552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076780.529787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913351552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076780.529813:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076780.529816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076780.529819:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913351552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076780.529824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913351680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076780.529841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076780.529850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076780/real 1769076780] req@ffff935002a61c00 x1855008913351680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076780.529858:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913351680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076780.529863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076780.529866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076780.529868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913351680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076780.529871:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913351808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076780.529881:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076780.529887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076780/real 1769076780] req@ffff935002a63100 x1855008913351808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076780.529894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913351808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076780.529899:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076780.529901:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076780.529902:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913351808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076780.529906:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076780.529915:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076780.529921:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076780/real 1769076780] req@ffff935002a62680 x1855008913351936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076780.529928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913351936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076780.529933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076780.529935:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076780.529937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913351936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076780.529956:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076780.529976:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076780.529988:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076780/real 1769076780] req@ffff935002a60a80 x1855008913352064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076780.530008:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913352064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076780.530030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076780.530032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076780.530035:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076780.530040:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076780.530055:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076780.530082:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076780/real 1769076780] req@ffff935002a61880 x1855008913352192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076780.530092:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913352192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076780.530098:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076780.530100:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076780.530102:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076780.721159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076781.233129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076781.233147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076781.233175:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076781.233177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076781.553147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076781.745167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076782.257185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076782.257185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076782.257234:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076782.257307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076782.577160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076782.769109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076783.281127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076783.281177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076783.281219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076783.281254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076783.601106:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076783.793140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076784.305165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076784.305185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076784.305264:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076784.305309:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076784.625118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076784.817174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076785.329133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076785.329135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076785.329146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076785.329224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076785.649139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076785.649199:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076785.649206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076785.649210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076785.649219:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2491 00000100:00080000:3.0:1769076785.649230:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076785.649236:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076785.649277:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913352320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076785.649301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076785.649304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076785.649306:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076785.649309:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2491 00000100:00080000:3.0:1769076785.649312:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076785.649316:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076785.649325:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076785.649336:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913352320:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076785.649337:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913352448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076785.649351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076785.649353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076785.649355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076785.649358:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2491 00000100:00080000:3.0:1769076785.649361:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076785.649363:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076785.649367:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076785.649374:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913352576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076785.649380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076785.649383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076785.649384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076785.649387:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2491 00000100:00080000:3.0:1769076785.649389:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076785.649392:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076785.649396:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076785.649402:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913352704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076785.649408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076785.649410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076785.649411:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076785.649415:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2491 00000100:00080000:3.0:1769076785.649419:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076785.649424:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076785.649430:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076785.649444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913352832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076785.649457:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913352448:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076785.649457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076785.649467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076785.649470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076785.649477:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2491 00000100:00080000:3.0:1769076785.649483:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076785.649488:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076785.649492:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913352576:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769076785.649495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076785.649509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913352960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076785.649516:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913352704:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076785.649521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076785.649525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076785.649527:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076785.649531:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2491 00000100:00080000:3.0:1769076785.649535:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076785.649540:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352832:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076785.649541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076785.649549:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076785.649561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913353088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076785.649569:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076785.649588:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076785.841201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076786.353233:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076786.353233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076786.353233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076786.353286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076786.673175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076786.865183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076787.377184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076787.377247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076787.377269:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076787.377327:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076787.697185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076787.889139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076788.401243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076788.401246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076788.401246:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076788.401287:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076788.721164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076788.721388:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076788.721449:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076788.721478:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076788.721559:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076788.721577:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076788.721594:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076788.721606:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076788.721607:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076785/real 1769076788] req@ffff935002a7c380 x1855008913352448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1769076788.721618:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076788.721634:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076788.721698:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913352448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076788.721720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076788.721727:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076788.721734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913352448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076788.721746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076785/real 1769076788] req@ffff935002a7ca80 x1855008913352576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076788.721768:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913352576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076788.721780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076788.721794:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076788.721799:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913352576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076788.721810:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076785/real 1769076788] req@ffff935002a7e300 x1855008913352704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076788.721829:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913352704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076788.721841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076788.721846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076788.721850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913352704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076788.721860:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076785/real 1769076788] req@ffff935002a7f100 x1855008913352320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076788.721877:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913352320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076788.721888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076788.721893:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076788.721897:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913352320:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076788.721938:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076785/real 1769076788] req@ffff935002a7c700 x1855008913353088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076788.721955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913353088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076788.721967:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076788.721972:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076788.721976:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076788.722004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076785/real 1769076788] req@ffff935002a7f800 x1855008913352832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076788.722019:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913352832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076788.722029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076788.722033:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076788.722037:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076788.722045:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076785/real 1769076788] req@ffff935002a7fb80 x1855008913352960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076788.722085:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913352960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076788.722095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076788.722100:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076788.722104:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913352960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076788.913161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076789.425127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076789.425143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076789.425175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076789.425185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076789.745186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076789.937131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076790.449160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076790.449160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076790.449160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076790.449198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076790.769123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076790.769152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076790.769157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076790.769161:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076790.769169:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2496 00000100:00080000:0.0:1769076790.769181:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076790.769188:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076790.769221:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913353216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076790.769245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076790.769248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076790.769250:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076790.769253:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2496 00000100:00080000:0.0:1769076790.769256:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076790.769260:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076790.769270:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076790.769280:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913353344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076790.769287:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913353216:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076790.769290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076790.769297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076790.769298:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076790.769302:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2496 00000100:00080000:0.0:1769076790.769304:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076790.769308:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076790.769311:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076790.769320:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913353472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076790.769330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076790.769334:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076790.769336:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076790.769341:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2496 00000100:00080000:0.0:1769076790.769345:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076790.769349:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076790.769355:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076790.769369:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913353600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076790.769380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076790.769388:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076790.769389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076790.769393:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076790.769399:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2496 00000100:00080000:0.0:1769076790.769404:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076790.769408:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076790.769414:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076790.769427:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913353728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076790.769439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076790.769440:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076790/real 1769076790] req@ffff9350103fbb80 x1855008913353216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076790.769448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076790.769453:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076790.769463:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2496 00000100:00100000:2.0:1769076790.769473:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913353216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076790.769481:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076790.769492:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076790.769500:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076790.769505:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076790.769515:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076790.769518:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913353856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076790.769523:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913353216:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076790.769539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076790.769544:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076790.769547:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076790.769552:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2496 00000100:00080000:0.0:1769076790.769557:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076790.769560:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913353344:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076790.769565:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076790.769572:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076790.769589:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913353984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076790.769605:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076790.769640:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076790/real 1769076790] req@ffff9350103fad80 x1855008913353344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076790.769653:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913353344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076790.769663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076790.769667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076790.769670:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913353344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076790.769678:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913353472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076790.769698:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076790.769711:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076790/real 1769076790] req@ffff9350103fb800 x1855008913353472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076790.769722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913353472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076790.769731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076790.769734:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076790.769737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913353472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076790.769743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913353600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076790.769762:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076790.769774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076790/real 1769076790] req@ffff9350103f9f80 x1855008913353600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076790.769786:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913353600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076790.769795:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076790.769799:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076790.769803:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913353600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076790.769809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076790.769830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076790.769842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076790/real 1769076790] req@ffff9350103f9500 x1855008913353728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076790.769855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913353728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076790.769864:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076790.769867:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076790.769871:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076790.769877:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076790.769897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076790.769911:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076790/real 1769076790] req@ffff9350103fb480 x1855008913353856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076790.769931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913353856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076790.769939:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076790.769943:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076790.769946:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076790.769971:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076790.769994:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076790.770008:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076790/real 1769076790] req@ffff9350103f9c00 x1855008913353984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076790.770023:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913353984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076790.770032:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076790.770036:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076790.770040:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913353984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076790.961141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076791.473127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076791.473127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076791.473127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076791.473144:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076791.793143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076791.985183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076792.497197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076792.497205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076792.497233:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076792.497258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076792.817113:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076793.009186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076793.521187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076793.521192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076793.521241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076793.521308:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076793.841158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076794.033219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076794.545229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076794.545230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076794.545233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076794.545273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076794.865214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076795.057171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076795.569125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076795.569204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076795.569247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076795.569251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076795.889173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076795.889179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076795.889199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076795.889205:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076795.889218:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2501 00000100:00080000:1.0:1769076795.889230:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076795.889237:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076795.889294:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913354112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076795.889333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076795.889337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076795.889340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076795.889345:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2501 00000100:00080000:1.0:1769076795.889349:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076795.889354:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076795.889365:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076795.889388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913354240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076795.889396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076795.889399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076795.889401:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076795.889404:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2501 00000100:00080000:1.0:1769076795.889407:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076795.889410:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076795.889415:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076795.889426:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913354112:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076795.889428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913354368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076795.889444:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076795.889448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076795.889450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076795.889455:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2501 00000100:00080000:1.0:1769076795.889471:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076795.889476:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076795.889481:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076795.889494:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913354496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076795.889501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076795.889504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076795.889505:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076795.889511:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2501 00000800:00000100:2.0:1769076795.889511:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076795.889517:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076795.889522:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076795.889527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076795.889551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913354624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076795.889565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076795.889567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076795.889569:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076795.889571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2501 00000100:00080000:1.0:1769076795.889574:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076795.889577:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076795.889582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076795.889587:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff9350100c8380 x1855008913354112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076795.889594:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913354752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076795.889607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076795.889610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076795.889611:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076795.889617:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2501 00000100:00100000:2.0:1769076795.889618:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913354112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076795.889619:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076795.889626:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076795.889632:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076795.889640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076795.889644:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913354880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076795.889649:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076795.889659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913354112:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076795.889671:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913354240:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076795.889699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076795.889714:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff9350100ca300 x1855008913354240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076795.889727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913354240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076795.889733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076795.889736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076795.889738:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913354240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076795.889759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913354368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076795.889775:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076795.889790:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff9350100ca680 x1855008913354368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076795.889799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913354368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076795.889808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076795.889811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076795.889813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913354368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076795.889817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913354496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076795.889838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076795.889844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff9350100c8e00 x1855008913354496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076795.889851:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913354496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076795.889862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076795.889864:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076795.889866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913354496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076795.889870:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913354624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076795.889881:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076795.889887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff9350100c9880 x1855008913354624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076795.889894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913354624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076795.889900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076795.889902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076795.889904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913354624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076795.889908:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913354752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076795.889926:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076795.889932:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff9350100c9500 x1855008913354752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076795.889939:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913354752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076795.889944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076795.889946:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076795.889947:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913354752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076795.889951:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913354880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076795.889960:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076795.889965:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff9350100caa00 x1855008913354880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076795.889974:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913354880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076795.889982:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076795.889985:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076795.889989:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913354880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076796.081136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076796.593178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076796.593184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076796.593187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076796.593256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076796.913215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076797.105135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076797.617208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076797.617222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076797.617311:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076797.617345:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076797.937276:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076798.129134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076798.641180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076798.641190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076798.641265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076798.641270:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076798.961168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076799.153135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076799.665141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076799.665195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076799.665197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076799.665242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076799.985151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076800.177155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076800.689160:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076800.689160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076800.689161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076800.689181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076801.009146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076801.009160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076801.009167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076801.009172:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076801.009200:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2506 00000100:00080000:1.0:1769076801.009214:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076801.009222:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076801.009268:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913355008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076801.009298:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076801.009302:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076801.009303:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076801.009308:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2506 00000100:00080000:1.0:1769076801.009311:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076801.009315:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076801.009326:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076801.009346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913355136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076801.009360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076801.009365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076801.009368:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076801.009374:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2506 00000100:00080000:1.0:1769076801.009381:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076801.009387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076801.009395:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076801.009400:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913355008:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076801.009419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913355264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076801.009443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076801.009469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076801.009473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076801.009480:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2506 00000100:00080000:1.0:1769076801.009507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076801.009516:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076801.009519:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076801.009535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076801.009567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913355392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076801.009580:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076801/real 1769076801] req@ffff9350100cb800 x1855008913355008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076801.009594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076801.009601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076801.009605:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076801.009611:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913355008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076801.009618:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2506 00000100:00080000:1.0:1769076801.009634:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076801.009645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076801.009648:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076801.009657:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769076801.009662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076801.009664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913355008:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076801.009684:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913355136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076801.009691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913355520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076801.009710:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076801.009719:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076801.009740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076801.009745:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076801.009755:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2506 00000100:00080000:1.0:1769076801.009763:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076801.009766:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076801/real 1769076801] req@ffff9350100caa00 x1855008913355136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076801.009778:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076801.009788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913355136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076801.009792:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076801.009797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076801.009802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076801.009806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913355136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076801.009833:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913355648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076801.009843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913355264:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769076801.009877:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076801.009880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076801.009883:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076801.009892:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2506 00000800:00000100:2.0:1769076801.009893:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076801.009908:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076801.009918:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076801.009919:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076801/real 1769076801] req@ffff9350100c9500 x1855008913355264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076801.009932:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076801.009938:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913355264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076801.009949:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076801.009950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913355776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076801.009965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076801.009979:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913355264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076801.009990:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913355392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076801.010006:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076801.010013:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076801/real 1769076801] req@ffff9350100c9880 x1855008913355392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076801.010020:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913355392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076801.010025:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076801.010027:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076801.010029:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913355392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076801.010033:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913355520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076801.010043:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076801.010092:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076801/real 1769076801] req@ffff9350100c8e00 x1855008913355520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076801.010106:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913355520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076801.010112:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076801.010114:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076801.010116:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913355520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076801.010128:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913355648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076801.010141:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076801.010148:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076801/real 1769076801] req@ffff9350100ca680 x1855008913355648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076801.010167:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913355648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076801.010172:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076801.010174:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076801.010176:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913355648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076801.010180:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913355776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076801.010189:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076801.010194:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076801/real 1769076801] req@ffff9350100ca300 x1855008913355776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076801.010202:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913355776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076801.010211:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076801.010213:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076801.010215:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913355776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076801.201167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076801.713178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076801.713187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076801.713190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076801.713277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076802.033162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076802.225134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076802.737203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076802.737203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076802.737220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076802.737299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076803.057178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076803.249150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076803.761143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076803.761164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076803.761194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076803.761268:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076804.081167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076804.273213:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076804.785139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076804.785139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076804.785146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076804.785289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076805.105191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076805.297139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076805.809143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076805.809143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076805.809163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076805.809330:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076806.129122:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076806.129177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076806.129183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076806.129187:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076806.129194:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2511 00000100:00080000:3.0:1769076806.129206:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076806.129212:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076806.129249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913355904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076806.129285:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076806.129290:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076806.129292:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076806.129297:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2511 00000100:00080000:3.0:1769076806.129301:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076806.129305:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076806.129316:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076806.129327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913356032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076806.129335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076806.129338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076806.129340:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076806.129345:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2511 00000100:00080000:3.0:1769076806.129349:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076806.129354:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076806.129360:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076806.129391:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913356160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076806.129395:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913355904:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076806.129412:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076806.129416:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076806.129418:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076806.129422:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2511 00000100:00080000:3.0:1769076806.129426:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076806.129429:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076806.129433:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076806.129445:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913356288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076806.129452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076806.129454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076806.129456:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076806.129458:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2511 00000100:00080000:3.0:1769076806.129468:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076806.129470:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076806.129474:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076806.129475:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076806.129486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913356416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076806.129494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076806.129497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076806.129499:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076806.129501:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2511 00000100:00080000:3.0:1769076806.129503:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076806.129505:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076806.129509:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076806.129517:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076806/real 1769076806] req@ffff935002a7c000 x1855008913355904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076806.129519:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913356544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076806.129551:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076806.129553:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913355904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076806.129555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076806.129559:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076806.129567:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2511 00000100:00080000:2.0:1769076806.129573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076806.129573:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076806.129579:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076806.129581:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769076806.129584:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076806.129586:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913355904:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076806.129595:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913356672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076806.129596:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076806.129624:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076806.129633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076806/real 1769076806] req@ffff935002a7df80 x1855008913356032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076806.129642:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913356032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076806.129648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076806.129651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076806.129653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076806.129673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076806.129684:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076806.129690:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076806/real 1769076806] req@ffff935002a7d180 x1855008913356160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076806.129697:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913356160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076806.129703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076806.129705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076806.129707:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076806.129711:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076806.129726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076806.129735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076806/real 1769076806] req@ffff935002a7dc00 x1855008913356288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076806.129748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913356288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076806.129757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076806.129760:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076806.129764:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076806.129771:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913356416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076806.129789:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076806.129797:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076806/real 1769076806] req@ffff935002a7fb80 x1855008913356416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076806.129805:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913356416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076806.129811:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076806.129814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076806.129816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913356416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076806.129820:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913356544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076806.129830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076806.129835:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076806/real 1769076806] req@ffff935002a7f800 x1855008913356544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076806.129843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913356544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076806.129848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076806.129850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076806.129852:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913356544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076806.129855:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913356672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076806.129864:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076806.129869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076806/real 1769076806] req@ffff935002a7c700 x1855008913356672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076806.129882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913356672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076806.129887:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076806.129889:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076806.129890:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913356672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076806.321124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076806.833140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076806.833170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076806.833217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076806.833251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076807.153130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076807.345135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076807.857223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076807.857223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076807.857239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076807.857282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076808.177134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076808.369103:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076808.881171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076808.881173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076808.881200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076808.881290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076809.201201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076809.393186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076809.905155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076809.905164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076809.905190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076809.905196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076810.225232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076810.417202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076810.929184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076810.929185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076810.929190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076810.929311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076811.249162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076811.249174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076811.249179:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076811.249188:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2516 00000100:00080000:2.0:1769076811.249200:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076811.249206:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076811.249237:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769076811.249260:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913356800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076811.249318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076811.249321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076811.249323:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076811.249327:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2516 00000100:00080000:2.0:1769076811.249330:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076811.249335:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076811.249344:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076811.249358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913356928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076811.249360:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913356800:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076811.249375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076811.249378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076811.249380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076811.249384:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2516 00000100:00080000:2.0:1769076811.249387:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076811.249390:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076811.249394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076811.249403:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913357056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076811.249410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076811.249413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076811.249414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076811.249416:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2516 00000100:00080000:2.0:1769076811.249418:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076811.249421:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076811.249424:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076811.249432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913357184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076811.249439:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076811.249443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076811.249447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076811.249448:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076811.249452:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2516 00000100:00080000:2.0:1769076811.249455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076811.249458:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076811.249461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076811.249469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913357312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076811.249476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076811.249479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076811.249480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076811.249483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2516 00000100:00080000:2.0:1769076811.249486:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076811.249491:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076811/real 1769076811] req@ffff935002a61500 x1855008913356800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076811.249491:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076811.249496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076811.249509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913357440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076811.249532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076811.249533:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913356800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076811.249543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076811.249548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076811.249552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076811.249556:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2516 00000100:00080000:0.0:1769076811.249559:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076811.249565:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913356800:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076811.249565:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076811.249575:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076811.249583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076811.249594:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076811.249598:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913357568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076811.249623:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076811.249637:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076811/real 1769076811] req@ffff935002a62300 x1855008913356928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076811.249646:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913356928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076811.249653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076811.249655:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076811.249658:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913356928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076811.249661:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076811.249672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076811.249678:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076811/real 1769076811] req@ffff935002a60000 x1855008913357056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076811.249687:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913357056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076811.249693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076811.249694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076811.249696:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076811.249700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076811.249719:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076811.249725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076811/real 1769076811] req@ffff935002a62a00 x1855008913357184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076811.249732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913357184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076811.249738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076811.249739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076811.249741:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076811.249744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913357312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076811.249754:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076811.249759:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076811/real 1769076811] req@ffff935002a61880 x1855008913357312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076811.249767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913357312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076811.249785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076811.249788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076811.249790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913357312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076811.249793:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913357440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076811.249802:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076811.249807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076811/real 1769076811] req@ffff935002a60a80 x1855008913357440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076811.249814:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913357440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076811.249819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076811.249821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076811.249823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913357440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076811.249846:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913357568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076811.249856:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076811.249862:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076811/real 1769076811] req@ffff935002a62680 x1855008913357568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076811.249870:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913357568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076811.249875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076811.249876:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076811.249879:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913357568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076811.441155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076811.953202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076811.953203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076811.953317:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076811.954515:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076812.273205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076812.465181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076812.977124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076812.977125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076812.977157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076812.977213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076813.297153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076813.489156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076814.001223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076814.001228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076814.001256:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076814.001279:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076814.321162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076814.513178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076815.025200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076815.025204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076815.025210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076815.025266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076815.345271:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076815.537115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076816.049148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076816.049148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076816.049158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076816.049220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076816.369232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076816.369266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076816.369276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076816.369282:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076816.369295:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2522 00000100:00080000:3.0:1769076816.369310:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076816.369320:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076816.369387:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913357696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076816.369429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076816.369435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076816.369437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076816.369444:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2522 00000100:00080000:3.0:1769076816.369450:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076816.369457:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076816.369472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076816.369477:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913357696:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076816.369495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913357824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076816.369512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076816.369532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076816.369535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076816.369541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2522 00000100:00080000:3.0:1769076816.369547:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076816.369552:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076816.369560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076816.369571:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076816.369579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913357952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076816.369594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076816.369600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076816.369602:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076816.369608:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2522 00000100:00080000:3.0:1769076816.369613:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076816.369618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076816.369626:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076816.369627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076816/real 1769076816] req@ffff935002a7f100 x1855008913357696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076816.369649:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913358080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076816.369651:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913357696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076816.369676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076816.369688:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076816.369692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076816.369694:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076816.369702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076816.369707:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2522 00000100:00100000:0.0:1769076816.369711:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913357696:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076816.369716:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076816.369725:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076816.369732:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076816.369744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076816.369752:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913358208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076816.369769:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076816.369770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076816.369777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076816.369780:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076816.369785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076816/real 1769076816] req@ffff935002a7e300 x1855008913357824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076816.369790:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2522 00000100:00080000:3.0:1769076816.369802:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076816.369807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913357824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076816.369814:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076816.369822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076816.369826:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076816.369829:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076816.369837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076816.369843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076816.369852:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913358336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076816.369863:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076816.369875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076816.369877:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076816/real 1769076816] req@ffff935002a7ca80 x1855008913357952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076816.369885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076816.369892:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076816.369894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913357952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076816.369905:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2522 00000100:00080000:0.0:1769076816.369907:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076816.369915:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076816.369917:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076816.369919:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913357952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076816.369930:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358080:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769076816.369930:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076816.369944:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076816.369952:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076816.369965:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076816/real 1769076816] req@ffff935002a7c380 x1855008913358080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076816.369975:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913358080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076816.369978:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913358464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076816.369981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076816.369986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076816.369993:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076816.370018:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913358208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076816.370034:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076816.370043:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076816/real 1769076816] req@ffff935002a7f100 x1855008913358208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076816.370082:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913358208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076816.370089:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076816.370091:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076816.370094:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913358208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076816.370099:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913358336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076816.370113:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076816.370139:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076816/real 1769076816] req@ffff935002a7c700 x1855008913358336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076816.370149:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913358336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076816.370156:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076816.370158:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076816.370161:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913358336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076816.370166:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913358464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076816.370180:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076816.370189:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076816/real 1769076816] req@ffff935002a7f800 x1855008913358464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076816.370217:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913358464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076816.370225:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076816.370227:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076816.370230:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913358464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076816.561216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076817.073207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076817.073207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076817.073245:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076817.073307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076817.393135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076817.585129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076818.097122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076818.097137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076818.097158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076818.097233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076818.417221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076818.609200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076819.121255:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076819.121255:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076819.121287:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076819.121313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076819.441185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076819.633195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076820.145151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076820.145151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076820.145153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076820.145194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076820.465226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076820.657226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076821.169219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076821.169219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076821.169227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076821.169247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076821.489157:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076821.489160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076821.489170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076821.489174:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076821.489182:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2527 00000100:00080000:0.0:1769076821.489194:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076821.489201:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076821.489238:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913358592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076821.489270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076821.489272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076821.489274:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076821.489277:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2527 00000100:00080000:0.0:1769076821.489280:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076821.489282:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076821.489291:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076821.489300:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913358720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076821.489308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076821.489311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076821.489313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076821.489316:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2527 00000100:00080000:0.0:1769076821.489318:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076821.489320:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076821.489324:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076821.489328:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913358592:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076821.489334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913358848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076821.489347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076821.489350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076821.489351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076821.489353:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2527 00000100:00080000:0.0:1769076821.489356:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076821.489358:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076821.489361:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076821.489369:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913358976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076821.489377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076821.489379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076821.489380:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076821.489395:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2527 00000100:00080000:0.0:1769076821.489397:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076821.489400:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076821.489402:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076821.489408:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913359104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076821.489414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076821.489417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076821.489418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076821.489421:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2527 00000100:00080000:0.0:1769076821.489424:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076821.489428:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076821.489429:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076821.489435:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076821.489444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913359232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076821.489455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076821.489458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076821.489460:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076821.489473:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2527 00000100:00080000:0.0:1769076821.489476:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076821.489479:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076821.489484:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076821/real 1769076821] req@ffff9350103fb100 x1855008913358592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076821.489485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076821.489498:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913359360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076821.489506:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913358592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076821.489516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076821.489520:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076821.489523:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913358592:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076821.489529:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358720:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076821.489545:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076821.489554:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076821/real 1769076821] req@ffff9350103f8380 x1855008913358720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076821.489562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913358720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076821.489590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076821.489592:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076821.489594:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076821.489615:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076821.489627:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076821.489633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076821/real 1769076821] req@ffff9350103f8700 x1855008913358848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076821.489652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913358848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076821.489658:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076821.489660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076821.489662:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076821.489665:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076821.489674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076821.489680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076821/real 1769076821] req@ffff9350103faa00 x1855008913358976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076821.489686:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913358976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076821.489692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076821.489694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076821.489697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913358976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076821.489701:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913359104:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076821.489712:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076821.489717:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076821/real 1769076821] req@ffff9350103f9180 x1855008913359104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076821.489725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913359104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076821.489732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076821.489735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076821.489739:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913359104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076821.489746:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913359232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076821.489766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076821.489777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076821/real 1769076821] req@ffff9350103f9c00 x1855008913359232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076821.489791:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913359232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076821.489800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076821.489804:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076821.489807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913359232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076821.489813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913359360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076821.489840:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076821.489848:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076821/real 1769076821] req@ffff9350103fb480 x1855008913359360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076821.489857:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913359360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076821.489862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076821.489864:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076821.489866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913359360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076821.681135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076822.193121:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076822.193127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076822.193130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076822.193173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076822.513225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076822.705203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076823.217201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076823.217202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076823.217235:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076823.217259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076823.537219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076823.729123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076824.241168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076824.241169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076824.241228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076824.241296:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076824.561202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076824.753200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076825.265227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076825.265229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076825.265243:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076825.265323:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076825.585199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076825.777184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076826.289135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076826.289153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076826.289189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076826.289243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076826.609191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076826.609196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076826.609208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076826.609214:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076826.609225:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2532 00000100:00080000:2.0:1769076826.609239:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076826.610918:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076826.610959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913359488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076826.610990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076826.610994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076826.610997:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076826.611001:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2532 00000100:00080000:2.0:1769076826.611006:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076826.611011:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076826.611040:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076826.611076:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913359488:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076826.611083:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913359616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076826.611097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076826.611101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076826.611104:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076826.611125:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2532 00000100:00080000:2.0:1769076826.611131:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076826.611136:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076826.611144:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076826.611159:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913359744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076826.611172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076826.611193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076826.611195:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076826.611203:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2532 00000800:00000100:0.0:1769076826.611208:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076826.611210:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076826.611217:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076826.611223:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076826.611237:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913359872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076826.611249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076826.611253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076826.611256:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076826.611260:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2532 00000100:00080000:2.0:1769076826.611264:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076826.611268:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076826.611274:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076826.611282:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076826/real 1769076826] req@ffff935002a63100 x1855008913359488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076826.611292:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913360000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076826.611333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076826.611338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076826.611342:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076826.611345:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913359488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076826.611364:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2532 00000100:00080000:2.0:1769076826.611390:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076826.611401:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076826.611414:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076826.611417:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076826.611429:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076826.611440:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913360128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076826.611441:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913359488:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076826.611465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076826.611471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076826.611473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076826.611478:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2532 00000100:00100000:0.0:1769076826.611484:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913359616:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076826.611486:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076826.611495:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076826.611503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076826.611519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913360256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076826.611525:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076826.611547:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076826/real 1769076826] req@ffff935002a61c00 x1855008913359616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076826.611567:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913359616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076826.611581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076826.611587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076826.611591:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913359616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076826.611602:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913359744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076826.611647:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076826.611661:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076826/real 1769076826] req@ffff935002a61f80 x1855008913359744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076826.611673:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913359744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076826.611691:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076826.611695:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076826.611698:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913359744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076826.611704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913359872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076826.611721:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076826.611729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076826/real 1769076826] req@ffff935002a61180 x1855008913359872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076826.611739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913359872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076826.611755:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076826.611758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076826.611760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913359872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076826.611765:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360000:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076826.611779:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076826.611797:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076826/real 1769076826] req@ffff935002a62680 x1855008913360000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076826.611808:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913360000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076826.611816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076826.611819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076826.611822:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076826.611827:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076826.611842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076826.611850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076826/real 1769076826] req@ffff935002a60a80 x1855008913360128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076826.611862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913360128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076826.611877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076826.611880:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076826.611882:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076826.611900:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076826.611914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076826.611942:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076826/real 1769076826] req@ffff935002a61880 x1855008913360256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076826.611956:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913360256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076826.611965:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076826.611969:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076826.611972:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076826.801115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076827.313201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076827.313201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076827.313249:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076827.313294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076827.633145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076827.825128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076828.337139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076828.337146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076828.337161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076828.337210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076828.657162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076828.849169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076829.361140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076829.361145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076829.361170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076829.361189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076829.681129:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076829.873168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076830.385168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076830.385170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076830.385172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076830.385281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076830.705147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076830.897161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076831.409137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076831.409166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076831.409175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076831.409211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076831.729134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076831.729143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076831.729151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076831.729155:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076831.729164:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2537 00000100:00080000:3.0:1769076831.729174:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076831.729181:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076831.729217:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913360384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076831.729244:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076831.729247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076831.729249:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076831.729253:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2537 00000100:00080000:3.0:1769076831.729256:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076831.729258:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076831.729267:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076831.729277:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913360512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076831.729287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076831.729290:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913360384:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076831.729291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076831.729294:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076831.729304:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2537 00000100:00080000:3.0:1769076831.729307:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076831.729309:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076831.729313:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076831.729324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913360640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076831.729331:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076831.729334:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076831.729335:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076831.729339:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2537 00000100:00080000:3.0:1769076831.729342:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076831.729344:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076831.729347:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076831.729355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913360768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076831.729362:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076831.729365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076831.729367:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076831.729369:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076831.729374:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2537 00000100:00080000:3.0:1769076831.729378:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076831.729382:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076831.729386:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076831.729396:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913360896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076831.729405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076831.729409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076831.729411:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076831.729415:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076831/real 1769076831] req@ffff935002a7fb80 x1855008913360384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076831.729418:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2537 00000100:00080000:3.0:1769076831.729424:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076831.729431:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076831.729438:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076831.729444:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913360384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076831.729450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913361024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076831.729467:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076831.729469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076831.729475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076831.729476:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076831.729479:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076831.729484:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913360384:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076831.729485:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2537 00000100:00080000:3.0:1769076831.729492:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076831.729496:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076831.729500:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076831.729508:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913361152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076831.729516:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913360512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076831.729539:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076831.729553:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076831/real 1769076831] req@ffff935002a7dc00 x1855008913360512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076831.729564:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913360512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076831.729571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076831.729574:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076831.729577:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913360512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076831.729582:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913360640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076831.729594:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076831.729601:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076831/real 1769076831] req@ffff935002a7d180 x1855008913360640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076831.729616:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913360640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076831.729622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076831.729625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076831.729627:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913360640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076831.729631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913360768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076831.729642:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076831.729649:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076831/real 1769076831] req@ffff935002a7df80 x1855008913360768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076831.729658:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913360768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076831.729664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076831.729667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076831.729669:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913360768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076831.729673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360896:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076831.729683:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076831.729690:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076831/real 1769076831] req@ffff935002a7c000 x1855008913360896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076831.729699:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913360896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076831.729711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076831.729713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076831.729716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913360896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076831.729719:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076831.729730:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076831.729737:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076831/real 1769076831] req@ffff935002a7f800 x1855008913361024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076831.729745:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913361024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076831.729750:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076831.729752:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076831.729755:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076831.729769:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076831.729780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076831.729788:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076831/real 1769076831] req@ffff935002a7c700 x1855008913361152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076831.729797:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913361152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076831.729803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076831.729805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076831.729808:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076831.921166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076832.433104:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076832.433122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076832.433137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076832.433165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076832.753147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076832.945123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076833.457196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076833.457198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076833.457207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076833.457268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076833.777127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076833.969120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076834.481148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076834.481163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076834.481164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076834.481196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076834.801168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076834.993112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076835.505149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076835.505172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076835.505207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076835.505219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076835.825199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076836.017152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076836.529192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076836.529197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076836.529269:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076836.529315:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076836.849152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076836.849152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076836.849167:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076836.849174:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076836.849184:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2542 00000100:00080000:3.0:1769076836.849199:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076836.849208:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076836.849251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913361280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076836.849288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076836.849293:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076836.849295:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076836.849300:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2542 00000100:00080000:3.0:1769076836.849303:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076836.849307:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076836.849317:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076836.849330:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913361408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076836.849342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076836.849346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076836.849350:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076836.849355:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2542 00000100:00080000:3.0:1769076836.849358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076836.849364:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913361280:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076836.849366:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076836.849378:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076836.849394:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913361536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076836.849405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076836.849408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076836.849411:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076836.849415:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2542 00000100:00080000:3.0:1769076836.849418:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076836.849422:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076836.849427:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076836.849438:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913361664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076836.849446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076836.849450:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076836.849452:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076836.849457:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2542 00000100:00080000:3.0:1769076836.849479:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076836.849489:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076836.849494:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076836.849499:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076836.849521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913361792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076836.849536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076836.849540:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076836.849542:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076836.849546:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2542 00000100:00080000:3.0:1769076836.849551:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076836.849556:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076836.849563:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076836.849581:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff935002a7f100 x1855008913361280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076836.849582:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913361920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076836.849635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076836.849643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076836.849646:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076836.849653:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2542 00000100:00080000:3.0:1769076836.849659:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076836.849665:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913361280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076836.849670:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076836.849684:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076836.849699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076836.849710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913362048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076836.849714:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076836.849727:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913361280:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076836.849758:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913361408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076836.849831:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076836.849881:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff935002a7c380 x1855008913361408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076836.849898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913361408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076836.849908:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076836.849913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076836.849917:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913361408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076836.849948:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913361536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076836.849973:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076836.849985:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff935002a7ca80 x1855008913361536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076836.849998:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913361536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076836.850007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076836.850011:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076836.850015:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913361536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076836.850020:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913361664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076836.850034:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076836.850043:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff935002a7e300 x1855008913361664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076836.850102:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913361664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076836.850112:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076836.850115:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076836.850118:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913361664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076836.850125:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361792:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076836.850145:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076836.850157:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff935002a7c700 x1855008913361792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076836.850169:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913361792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076836.850177:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076836.850180:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076836.850216:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076836.850225:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076836.850250:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076836.850264:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff935002a7f800 x1855008913361920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076836.850278:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913361920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076836.850289:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076836.850293:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076836.850297:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913361920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076836.850303:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076836.850321:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076836.850332:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff935002a7c000 x1855008913362048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076836.850356:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913362048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076836.850365:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076836.850368:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076836.850372:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076837.041149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076837.553134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076837.553183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076837.553193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076837.553216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076837.873249:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076838.065202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076838.577164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076838.577164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076838.577165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076838.577197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076838.897192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076839.089117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076839.601183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076839.601196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076839.601241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076839.601273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076839.921256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076840.113217:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076840.625229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076840.625264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076840.625271:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076840.625303:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076840.945119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076841.137127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076841.649165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076841.649179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076841.649232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076841.649286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076841.969219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076841.969280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.969291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076841.969298:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.969311:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2547 00000100:00080000:0.0:1769076841.969326:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.969335:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076841.969382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913362176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076841.969421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.969426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076841.969429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.969434:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2547 00000100:00080000:0.0:1769076841.969439:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.969443:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076841.969457:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076841.969473:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913362304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076841.969488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.969493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076841.969495:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.969501:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2547 00000100:00080000:0.0:1769076841.969505:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.969509:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076841.969516:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076841.969520:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913362176:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076841.969540:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913362432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076841.969571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.969578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076841.969582:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.969590:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2547 00000100:00080000:0.0:1769076841.969599:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.969607:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076841.969631:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769076841.969640:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076841.969659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913362560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076841.969682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.969689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076841.969694:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076841.969701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9350103f9500 x1855008913362176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076841.969721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2547 00000100:00080000:0.0:1769076841.969735:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.969749:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076841.969755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913362176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076841.969789:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076841.969824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076841.969829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913362688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076841.969838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076841.969855:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913362176:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076841.969861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.969869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076841.969873:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076841.969876:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913362304:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076841.969883:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2547 00000100:00080000:0.0:1769076841.969893:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.969899:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076841.969905:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076841.969924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913362816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076841.969935:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076841.969950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.969956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076841.969959:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.969967:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2547 00000100:00100000:2.0:1769076841.969968:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9350103f9f80 x1855008913362304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076841.969973:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.969985:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076841.969999:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076841.970009:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913362304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076841.970020:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913362944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076841.970040:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.970115:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076841.970124:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913362304:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076841.970170:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913362432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076841.970203:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076841.970222:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9350103fb800 x1855008913362432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.970253:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913362432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.970265:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.970269:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076841.970273:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913362432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076841.970282:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913362560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076841.970303:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076841.970316:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9350103fad80 x1855008913362560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.970330:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913362560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.970341:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.970345:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076841.970350:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913362560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076841.970361:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362688:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076841.970395:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076841.970414:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9350103fbb80 x1855008913362688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.970440:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913362688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.970458:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.970464:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076841.970471:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076841.970483:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076841.970516:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076841.970536:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9350103fb480 x1855008913362816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.970551:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913362816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.970562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.970566:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076841.970570:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076841.970577:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076841.970596:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076841.970615:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9350103f9c00 x1855008913362944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.970630:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913362944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.970639:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.970642:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076841.970646:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913362944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076842.161170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076842.673141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076842.673147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076842.673195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076842.673224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076842.993255:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076843.185174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076843.697180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076843.697191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076843.697225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076843.697253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076844.017179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076844.209160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076844.721155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076844.721155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076844.721172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076844.721220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076845.041149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076845.233153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076845.745160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076845.745184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076845.745186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076845.745207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076846.065159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076846.257178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076846.769185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076846.769185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076846.769185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076846.769248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076847.089129:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076847.089194:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076847.089202:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076847.089206:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076847.089214:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2552 00000100:00080000:2.0:1769076847.089225:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076847.089230:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076847.089274:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913363072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076847.089300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076847.089303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076847.089305:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076847.089308:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2552 00000100:00080000:2.0:1769076847.089310:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076847.089314:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076847.089322:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076847.089332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913363200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076847.089340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076847.089342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076847.089343:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076847.089346:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2552 00000100:00080000:2.0:1769076847.089348:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076847.089355:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076847.089356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913363072:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076847.089359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076847.089375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913363328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076847.089381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076847.089384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076847.089386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076847.089389:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2552 00000100:00080000:2.0:1769076847.089391:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076847.089394:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076847.089398:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076847.089405:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913363456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076847.089411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076847.089414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076847.089415:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076847.089418:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2552 00000100:00080000:2.0:1769076847.089422:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076847.089425:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076847.089428:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076847.089436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913363584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076847.089442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076847.089444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076847.089445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076847.089448:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2552 00000100:00080000:2.0:1769076847.089450:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076847.089456:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076847.089462:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076847.089467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076847.089476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913363712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076847.089484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076847.089486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076847.089488:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076847.089490:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2552 00000100:00080000:2.0:1769076847.089492:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076847.089494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076847.089499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076847.089509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913363840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076847.089518:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076847/real 1769076847] req@ffff935002a62a00 x1855008913363072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076847.089544:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913363072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076847.089557:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076847.089561:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076847.089566:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913363072:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076847.089596:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913363200:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076847.089620:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076847.089633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076847/real 1769076847] req@ffff935002a60000 x1855008913363200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076847.089646:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913363200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076847.089657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076847.089661:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076847.089665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913363200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076847.089672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913363328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076847.089711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076847.089725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076847/real 1769076847] req@ffff935002a62300 x1855008913363328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076847.089739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913363328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076847.089748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076847.089751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076847.089754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913363328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076847.089759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913363456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076847.089784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076847.089792:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076847/real 1769076847] req@ffff935002a61500 x1855008913363456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076847.089801:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913363456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076847.089807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076847.089809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076847.089812:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913363456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076847.089816:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913363584:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076847.089826:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076847.089833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076847/real 1769076847] req@ffff935002a61880 x1855008913363584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076847.089841:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913363584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076847.089847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076847.089849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076847.089852:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913363584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076847.089858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913363712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076847.089878:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076847.089890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076847/real 1769076847] req@ffff935002a60a80 x1855008913363712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076847.089903:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913363712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076847.089912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076847.089916:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076847.089920:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913363712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076847.089926:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913363840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076847.089941:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076847.089948:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076847/real 1769076847] req@ffff935002a62680 x1855008913363840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076847.089956:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913363840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076847.089962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076847.089964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076847.089967:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913363840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076847.281154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076847.793149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076847.793193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076847.793202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076847.793237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076848.113169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076848.305129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076848.817209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076848.817209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076848.817210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076848.817248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076849.137128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076849.329145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076849.841131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076849.841144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076849.841159:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076849.841179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076850.161170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076850.353135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076850.865155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076850.865155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076850.865174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076850.865180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076851.185120:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076851.377169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076851.889152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076851.889153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076851.889153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076851.889178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076852.209176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076852.209183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076852.209196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076852.209202:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076852.209229:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2557 00000100:00080000:2.0:1769076852.209245:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076852.209255:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076852.209310:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913363968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076852.209368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076852.209375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076852.209379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076852.209387:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2557 00000100:00080000:2.0:1769076852.209393:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076852.209399:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076852.209426:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076852.209449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913364096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076852.209474:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913363968:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076852.209476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076852.209488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076852.209495:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076852.209501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2557 00000100:00080000:2.0:1769076852.209506:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076852.209520:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076852.209528:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076852.209543:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913364224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076852.209555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076852.209559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076852.209562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076852.209565:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2557 00000100:00080000:2.0:1769076852.209569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076852.209572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076852.209577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076852.209617:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913364352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076852.209634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076852.209641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076852.209645:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076852.209653:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2557 00000100:00080000:2.0:1769076852.209662:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076852.209672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364096:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076852.209674:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076852.209692:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076852.209720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913364480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076852.209731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364224:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076852.209743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076852.209750:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076852.209753:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076852.209760:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2557 00000100:00080000:2.0:1769076852.209767:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076852.209768:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364352:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076852.209778:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076852.209787:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076852.209807:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913364608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076852.209811:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913364480:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076852.209849:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076852.209856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913364608:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769076852.209857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076852.209863:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076852.209872:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2557 00000100:00080000:2.0:1769076852.209879:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076852.209885:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076852.209893:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076852.209909:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913364736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076852.210001:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913364736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076852.401136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076852.913160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076852.913161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076852.913187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076852.913225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076853.233323:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076853.425174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076853.937176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076853.937196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076853.937249:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076853.937271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076854.257319:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076854.449147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076854.961136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076854.961151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076854.961165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076854.961203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076855.281370:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076855.281578:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076855.281628:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076855.281650:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076855.281708:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076855.281720:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076855.281732:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076855.281745:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076855.281756:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076855.281768:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076855.281784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076852/real 1769076855] req@ffff935002a60a80 x1855008913364608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076855.281826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913364608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076855.281842:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076855.281850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076855.281856:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913364608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076855.281868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076852/real 1769076855] req@ffff935002a62680 x1855008913364480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076855.281884:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913364480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076855.281895:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076855.281900:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076855.281905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913364480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076855.281930:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076852/real 1769076855] req@ffff935002a61f80 x1855008913364096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076855.281946:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913364096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076855.281955:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076855.281960:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076855.281965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076855.281973:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076852/real 1769076855] req@ffff935002a61c00 x1855008913364224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076855.281986:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913364224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076855.281996:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076855.282000:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076855.282004:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076855.282013:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076852/real 1769076855] req@ffff935002a63100 x1855008913364352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076855.282027:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913364352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076855.282035:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076855.282040:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076855.282045:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076855.282087:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076852/real 1769076855] req@ffff935002a61180 x1855008913363968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076855.282104:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913363968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076855.282112:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076855.282116:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076855.282119:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913363968:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076855.282160:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076852/real 1769076855] req@ffff935002a61880 x1855008913364736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076855.282176:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913364736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076855.282193:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076855.282197:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076855.282201:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913364736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076855.473145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076855.985132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076855.985136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076855.985136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076855.985172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076856.305207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076856.497109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076857.009188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076857.009198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076857.009202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076857.009323:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076857.329188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076857.329204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076857.329210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076857.329222:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2563 00000100:00080000:2.0:1769076857.329254:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076857.329261:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076857.329299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913364864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076857.329320:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076857.329343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076857.329349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076857.329352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076857.329357:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2563 00000100:00080000:2.0:1769076857.329362:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076857.329367:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076857.329377:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076857.329389:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913364992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076857.329399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076857.329403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076857.329404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076857.329408:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2563 00000100:00080000:2.0:1769076857.329411:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076857.329413:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076857.329420:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913364864:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076857.329422:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076857.329437:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913365120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076857.329447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076857.329451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076857.329452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076857.329456:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2563 00000100:00080000:2.0:1769076857.329458:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076857.329461:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076857.329465:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076857.329482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913365248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076857.329494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076857.329500:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076857.329521:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076857.329527:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2563 00000100:00080000:2.0:1769076857.329533:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076857.329542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769076857.329544:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769076857.329555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076857.329578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913365376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076857.329592:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076857.329597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076857.329600:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076857.329606:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076857/real 1769076857] req@ffff935002a61500 x1855008913364864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076857.329609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2563 00000100:00080000:2.0:1769076857.329618:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076857.329629:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076857.329640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076857.329649:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913364864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076857.329660:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913365504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076857.329670:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076857.329681:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076857.329683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076857.329692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076857.329696:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913364864:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076857.329698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076857.329722:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2563 00000100:00100000:0.0:1769076857.329729:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364992:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076857.329731:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076857.329743:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076857.329753:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076857.329772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913365632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076857.329784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076857.329815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076857/real 1769076857] req@ffff935002a62300 x1855008913364992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076857.329836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913364992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076857.329849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076857.329855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076857.329859:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913364992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076857.329889:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913365120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076857.329920:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076857.329936:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076857/real 1769076857] req@ffff935002a60000 x1855008913365120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076857.329953:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913365120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076857.329964:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076857.329969:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076857.329975:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913365120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076857.329984:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913365248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076857.330025:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076857.330043:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076857/real 1769076857] req@ffff935002a62a00 x1855008913365248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076857.330099:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913365248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076857.330112:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076857.330117:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076857.330121:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913365248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076857.330132:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913365376:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076857.330164:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076857.330181:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076857/real 1769076857] req@ffff935002a61880 x1855008913365376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076857.330198:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913365376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076857.330209:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076857.330214:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076857.330218:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913365376:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076857.330228:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913365504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076857.330276:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076857.330292:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076857/real 1769076857] req@ffff935002a61180 x1855008913365504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076857.330309:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913365504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076857.330321:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076857.330325:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076857.330330:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913365504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076857.330338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913365632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076857.330367:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076857.330387:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076857/real 1769076857] req@ffff935002a63100 x1855008913365632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076857.330418:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913365632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076857.330430:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076857.330435:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076857.330441:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913365632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076857.521160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076858.033182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076858.033182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076858.033247:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076858.033254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076858.353307:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076858.545119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076859.057218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076859.057219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076859.057315:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076859.057367:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076859.377210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076859.569179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076860.081232:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076860.081236:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076860.081295:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076860.081301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076860.401187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076860.593172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076861.105226:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076861.105260:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076861.105290:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076861.105320:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076861.425215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076861.617156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076862.129171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076862.129208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076862.129208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076862.129210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076862.449211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076862.449249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076862.449262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076862.449269:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076862.449282:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2568 00000100:00080000:0.0:1769076862.449297:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076862.449306:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076862.449376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913365760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076862.449440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076862.449446:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076862.449449:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076862.449455:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2568 00000100:00080000:0.0:1769076862.449460:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076862.449483:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076862.449497:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076862.449505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913365760:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076862.449516:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913365888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076862.449561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076862.449566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076862.449570:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076862.449576:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2568 00000100:00080000:0.0:1769076862.449581:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076862.449585:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076862.449592:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076862.449604:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913366016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076862.449615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076862.449630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076862.449632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076862.449646:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2568 00000800:00000100:2.0:1769076862.449647:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076862.449658:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076862.449681:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076862.449692:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076862.449703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076862/real 1769076862] req@ffff9350103f9180 x1855008913365760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076862.449719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913366144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076862.449733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913365760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076862.449786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076862.449788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076862.449802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076862.449802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076862.449812:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913365760:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076862.449813:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076862.449839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2568 00000100:00080000:0.0:1769076862.449850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076862.449858:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076862.449865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913365888:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769076862.449871:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076862.449897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913366272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076862.449897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076862.449915:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076862/real 1769076862] req@ffff9350103faa00 x1855008913365888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076862.449916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076862.449936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076862.449947:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076862.449954:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913365888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076862.449955:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2568 00000100:00080000:0.0:1769076862.449965:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076862.449975:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076862.449987:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076862.449993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076862.450006:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913366400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076862.450009:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076862.450026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076862.450029:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913365888:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076862.450031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076862.450041:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076862.450046:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2568 00000100:00080000:0.0:1769076862.450050:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076862.450051:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076862.450093:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076862.450106:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076862/real 1769076862] req@ffff9350103f8700 x1855008913366016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076862.450115:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076862.450130:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076862.450164:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913366016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076862.450167:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913366528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076862.450188:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076862.450200:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076862.450226:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076862.450236:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076862.450258:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076862.450266:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076862/real 1769076862] req@ffff9350103f8380 x1855008913366144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076862.450276:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913366144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076862.450282:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076862.450284:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076862.450287:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076862.450302:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913366272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076862.450314:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076862.450321:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076862/real 1769076862] req@ffff9350103fb100 x1855008913366272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076862.450330:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913366272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076862.450363:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076862.450366:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076862.450370:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913366272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076862.450376:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913366400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076862.450393:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076862.450401:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076862/real 1769076862] req@ffff9350103f9180 x1855008913366400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076862.450413:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913366400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076862.450485:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076862.450488:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076862.450507:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913366400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076862.450513:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913366528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076862.450530:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076862.450538:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076862/real 1769076862] req@ffff9350103f9c00 x1855008913366528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076862.450547:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913366528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076862.450553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076862.450556:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076862.450558:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913366528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076862.641251:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076863.153188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076863.153222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076863.153244:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076863.153309:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076863.473221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076863.665157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076864.177187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076864.177294:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076864.177330:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076864.177333:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076864.497201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076864.689164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076865.201196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076865.201232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076865.201293:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076865.201327:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076865.521164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076865.713183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076866.225227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076866.225268:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076866.225277:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076866.225389:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076866.545267:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076866.737151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076867.249176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076867.249271:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076867.249380:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076867.249391:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076867.569184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076867.569185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076867.569198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076867.569204:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076867.569218:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2573 00000100:00080000:3.0:1769076867.569233:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076867.569244:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076867.569291:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913366656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076867.569321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076867.569325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076867.569327:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076867.569346:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2573 00000100:00080000:3.0:1769076867.569352:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076867.569356:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076867.569366:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076867.569379:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913366784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076867.569389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076867.569393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076867.569395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076867.569400:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2573 00000100:00080000:3.0:1769076867.569403:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076867.569406:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076867.569414:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076867.569425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913366912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076867.569440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076867.569443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076867.569445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076867.569448:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2573 00000100:00080000:3.0:1769076867.569451:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076867.569460:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076867.569466:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913366656:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076867.569468:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076867.569494:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913367040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076867.569512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076867.569518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076867.569521:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076867.569529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2573 00000100:00080000:3.0:1769076867.569536:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076867.569545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076867.569555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076867.569578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913367168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076867.569589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076867.569596:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076867.569602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076867.569608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076867.569616:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2573 00000100:00080000:3.0:1769076867.569623:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076867.569629:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076867.569639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076867.569660:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913367296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076867.569674:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076867/real 1769076867] req@ffff935002a7df80 x1855008913366656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076867.569703:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076867.569709:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076867.569712:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076867.569718:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2573 00000100:00100000:2.0:1769076867.569723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913366656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076867.569729:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076867.569752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076867.569752:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076867.569770:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076867.569777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913366656:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076867.569778:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076867.569794:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076867.569825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913367424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076867.569833:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076867.569868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076867/real 1769076867] req@ffff935002a7d180 x1855008913366784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076867.569896:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913366784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076867.569917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076867.569925:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076867.569932:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076867.569984:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076867.570021:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076867.570038:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076867/real 1769076867] req@ffff935002a7dc00 x1855008913366912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076867.570102:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913366912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076867.570114:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076867.570118:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076867.570122:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913366912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076867.570131:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076867.570156:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076867.570169:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076867/real 1769076867] req@ffff935002a7fb80 x1855008913367040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076867.570183:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913367040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076867.570193:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076867.570197:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076867.570201:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076867.570208:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913367168:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076867.570227:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076867.570237:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076867/real 1769076867] req@ffff935002a7c000 x1855008913367168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076867.570252:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913367168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076867.570263:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076867.570266:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076867.570270:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913367168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076867.570277:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913367296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076867.570295:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076867.570306:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076867/real 1769076867] req@ffff935002a7f800 x1855008913367296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076867.570320:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913367296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076867.570364:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076867.570370:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076867.570375:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913367296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076867.570383:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913367424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076867.570412:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076867.570427:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076867/real 1769076867] req@ffff935002a7c700 x1855008913367424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076867.570442:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913367424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076867.570461:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076867.570465:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076867.570469:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913367424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076867.761132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076868.273186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076868.273260:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076868.273314:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076868.273338:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076868.593148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076868.785194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076869.297206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076869.297244:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076869.297259:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076869.297391:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076869.617189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076869.809158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076870.321180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076870.321181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076870.321191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076870.321234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076870.641189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076870.833199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076871.345247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076871.345248:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076871.345332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076871.345337:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076871.665184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076871.857150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076872.369216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076872.369226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076872.369270:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076872.369318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076872.689310:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076872.689370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076872.689386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076872.689393:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076872.689408:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2578 00000100:00080000:0.0:1769076872.689424:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076872.689435:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076872.689514:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913367552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076872.689568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076872.689576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076872.689579:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076872.689588:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2578 00000100:00080000:0.0:1769076872.689595:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076872.689605:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076872.689624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076872.689651:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913367552:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076872.689653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913367680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076872.689684:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076872.689692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076872.689696:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076872.689705:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2578 00000100:00080000:0.0:1769076872.689711:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076872.689720:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076872.689730:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076872.689754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913367808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076872.689774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076872.689779:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076872.689781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076872.689788:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076872.689796:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2578 00000100:00080000:0.0:1769076872.689802:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076872.689808:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076872.689816:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076872.689833:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913367936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076872.689844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076872.689849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076872.689851:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076872.689856:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2578 00000100:00080000:0.0:1769076872.689874:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076872.689883:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076872.689884:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076872/real 1769076872] req@ffff9350103fb480 x1855008913367552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076872.689890:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076872.689914:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913368064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076872.689928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076872.689929:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913367552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076872.689939:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076872.689945:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076872.689958:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2578 00000100:00080000:2.0:1769076872.689962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076872.689964:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076872.689976:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076872.689980:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:0.0:1769076872.689985:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076872.689991:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913367552:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076872.690006:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913368192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076872.690020:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076872.690024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076872.690026:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076872.690032:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2578 00000100:00080000:0.0:1769076872.690036:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076872.690040:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367680:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076872.690046:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076872.690109:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076872.690129:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076872.690132:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913368320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076872.690175:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076872/real 1769076872] req@ffff9350103fbb80 x1855008913367680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076872.690195:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913367680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076872.690207:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076872.690212:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076872.690217:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076872.690225:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076872.690273:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076872.690290:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076872/real 1769076872] req@ffff9350103fad80 x1855008913367808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076872.690305:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913367808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076872.690314:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076872.690318:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076872.690322:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076872.690328:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076872.690352:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076872.690362:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076872/real 1769076872] req@ffff9350103fb800 x1855008913367936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076872.690376:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913367936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076872.690385:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076872.690389:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076872.690392:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913367936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076872.690398:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368064:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076872.690416:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076872.690426:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076872/real 1769076872] req@ffff9350103f9f80 x1855008913368064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076872.690440:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913368064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076872.690449:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076872.690452:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076872.690456:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076872.690462:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076872.690478:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076872.690487:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076872/real 1769076872] req@ffff9350103f9500 x1855008913368192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076872.690500:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913368192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076872.690510:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076872.690513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076872.690516:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076872.690538:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076872.690555:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076872.690571:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076872/real 1769076872] req@ffff9350103fb480 x1855008913368320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076872.690606:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913368320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076872.690622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076872.690628:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076872.690635:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076872.881215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076873.393171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076873.393171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076873.393182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076873.393212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076873.713199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076873.905159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076874.417214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076874.417214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076874.417214:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076874.417297:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076874.737186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076874.929180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076875.441231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076875.441235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076875.441262:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076875.441276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076875.761176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076875.953144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076876.465231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076876.465261:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076876.465270:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076876.465283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076876.785171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076876.977206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076877.489234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076877.489234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076877.489300:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076877.489312:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076877.809133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076877.809142:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076877.809146:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076877.809154:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2583 00000100:00080000:1.0:1769076877.809181:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076877.809187:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076877.809218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769076877.809249:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913368448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076877.809303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076877.809308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076877.809310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076877.809315:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2583 00000100:00080000:1.0:1769076877.809320:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076877.809325:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076877.809335:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076877.809340:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913368448:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076877.809354:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913368576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076877.809364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076877.809367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076877.809368:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076877.809372:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2583 00000100:00080000:1.0:1769076877.809375:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076877.809379:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076877.809383:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076877.809392:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913368704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076877.809398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076877.809401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076877.809404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076877.809405:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076877.809409:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2583 00000100:00080000:1.0:1769076877.809412:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076877.809415:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076877.809418:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076877.809426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913368832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076877.809433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076877.809435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076877.809437:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076877.809440:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2583 00000100:00100000:2.0:1769076877.809443:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076877/real 1769076877] req@ffff9350100c8380 x1855008913368448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076877.809445:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076877.809451:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076877.809458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076877.809470:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913368448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076877.809472:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913368960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076877.809488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076877.809490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076877.809491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076877.809494:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2583 00000100:00080000:1.0:1769076877.809499:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076877.809502:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076877.809506:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076877.809506:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076877.809515:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769076877.809519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913369088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076877.809523:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913368448:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076877.809535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076877.809538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076877.809539:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076877.809542:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2583 00000100:00080000:1.0:1769076877.809545:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076877.809548:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076877.809551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076877.809552:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913368576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076877.809579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913369216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076877.809613:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076877.809638:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076877/real 1769076877] req@ffff9350100c9f80 x1855008913368576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076877.809679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913368576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076877.809686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076877.809689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076877.809691:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913368576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076877.809696:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913368704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076877.809712:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076877.809720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076877/real 1769076877] req@ffff9350100cad80 x1855008913368704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076877.809728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913368704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076877.809733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076877.809735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076877.809737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913368704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076877.809741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913368832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076877.809751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076877.809756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076877/real 1769076877] req@ffff9350100c8000 x1855008913368832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076877.809764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913368832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076877.809775:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076877.809777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076877.809779:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913368832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076877.809783:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368960:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076877.809791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076877.809796:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076877/real 1769076877] req@ffff9350100cb100 x1855008913368960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076877.809803:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913368960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076877.809810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076877.809829:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076877.809835:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913368960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076877.809845:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076877.809892:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076877.809908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076877/real 1769076877] req@ffff9350100c9180 x1855008913369088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076877.809928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913369088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076877.809957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076877.809960:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076877.809963:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076877.809978:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076877.809992:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076877.809999:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076877/real 1769076877] req@ffff9350100c9c00 x1855008913369216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076877.810024:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913369216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076877.810037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076877.810041:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076877.810046:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076878.001137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076878.513132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076878.513134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076878.513162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076878.513197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076878.833248:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076879.025164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076879.537230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076879.537230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076879.537230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076879.537485:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076879.857196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076880.049227:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076880.561190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076880.561191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076880.561195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076880.561331:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076880.881183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076881.073230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076881.585150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076881.585173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076881.585175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076881.585216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076881.905178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076882.097146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076882.609132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076882.609136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076882.609152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076882.609186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076882.929217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076882.929217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076882.929230:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076882.929236:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076882.929247:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2588 00000100:00080000:3.0:1769076882.929262:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076882.929270:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076882.929314:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913369344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076882.929356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076882.929361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076882.929364:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076882.929371:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2588 00000100:00080000:3.0:1769076882.929377:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076882.929384:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076882.929399:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076882.929414:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913369344:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076882.929419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913369472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076882.929434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076882.929437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076882.929439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076882.929444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2588 00000100:00080000:3.0:1769076882.929449:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076882.929452:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076882.929457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076882.929470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913369600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076882.929479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076882.929483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076882.929485:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076882.929488:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2588 00000100:00080000:3.0:1769076882.929497:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076882.929502:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076882.929509:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076882.929513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076882.929535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913369728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076882.929551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076882.929556:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076882.929558:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076882.929566:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff935002a7e300 x1855008913369344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076882.929567:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2588 00000100:00080000:3.0:1769076882.929579:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076882.929591:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076882.929599:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913369344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076882.929600:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076882.929620:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076882.929626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769076882.929630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913369856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076882.929636:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913369344:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076882.929654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076882.929658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076882.929661:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076882.929665:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2588 00000100:00080000:3.0:1769076882.929670:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076882.929673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913369472:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076882.929679:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076882.929688:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076882.929703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913369984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076882.929713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076882.929714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076882.929722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076882.929725:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076882.929730:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2588 00000100:00080000:3.0:1769076882.929735:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076882.929738:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff935002a7ca80 x1855008913369472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076882.929742:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076882.929752:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076882.929768:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913369472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076882.929770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913370112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076882.929805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076882.929813:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076882.929819:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913369472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076882.929826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913369600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076882.929855:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076882.929871:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff935002a7c380 x1855008913369600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076882.929908:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913369600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076882.929925:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076882.929931:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076882.929938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913369600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076882.929949:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913369728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076882.929981:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076882.929995:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff935002a7f100 x1855008913369728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076882.930026:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913369728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076882.930036:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076882.930040:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076882.930043:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913369728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076882.930050:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369856:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076882.930067:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076882.930143:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff935002a7c700 x1855008913369856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076882.930159:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913369856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076882.930168:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076882.930172:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076882.930175:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076882.930227:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076882.930257:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076882.930274:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff935002a7f800 x1855008913369984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076882.930293:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913369984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076882.930305:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076882.930310:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076882.930314:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913369984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076882.930320:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913370112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076882.930340:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076882.930351:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff935002a7c000 x1855008913370112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076882.930362:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913370112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076882.930369:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076882.930372:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076882.930375:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913370112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076883.121170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076883.633223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076883.633247:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076883.633248:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076883.633332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076883.953205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076884.145150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076884.657156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076884.657163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076884.657165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076884.657239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076884.977229:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076885.169205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076885.681188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076885.681217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076885.681218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076885.681260:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076886.001163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076886.193168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076886.705152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076886.705223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076886.705225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076886.705243:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076887.025243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076887.217157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076887.729251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076887.729251:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076887.729252:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076887.729299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076888.049298:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076888.049377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076888.049392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076888.049400:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076888.049417:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2593 00000100:00080000:1.0:1769076888.049438:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076888.049467:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076888.049552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913370240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076888.049602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076888.049608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076888.049611:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076888.049617:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2593 00000100:00080000:1.0:1769076888.049622:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076888.049628:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076888.049642:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076888.049664:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913370368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076888.049677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076888.049681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076888.049684:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076888.049688:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2593 00000100:00080000:1.0:1769076888.049692:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076888.049697:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076888.049703:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076888.049717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913370496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076888.049731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076888.049737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076888.049739:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076888.049746:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2593 00000100:00080000:1.0:1769076888.049750:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076888.049757:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913370240:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076888.049760:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076888.049773:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076888.049787:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913370624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076888.049818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076888.049821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076888.049824:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076888.049829:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2593 00000100:00080000:1.0:1769076888.049834:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076888.049840:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076888.049847:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076888.049861:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913370752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076888.049899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076888.049903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076888.049907:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076888.049917:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2593 00000800:00000100:2.0:1769076888.049920:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076888.049925:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076888.049933:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076888.049940:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076888.049958:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913370880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076888.049970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076888.049974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076888.049977:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076888.049982:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2593 00000100:00100000:2.0:1769076888.050097:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076888/real 1769076888] req@ffff9350100c8a80 x1855008913370240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076888.050132:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076888.050148:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076888.050158:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076888.050172:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913370240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076888.050189:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913371008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076888.050194:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076888.050212:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076888.050226:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913370240:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076888.050252:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913370368:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076888.050312:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076888.050348:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076888/real 1769076888] req@ffff9350100c9c00 x1855008913370368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076888.050375:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913370368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076888.050395:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076888.050402:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076888.050407:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913370368:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076888.050416:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913370496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076888.050446:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076888.050478:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076888/real 1769076888] req@ffff9350100c9180 x1855008913370496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076888.050493:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913370496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076888.050503:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076888.050507:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076888.050511:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913370496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076888.050562:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913370624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076888.050584:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076888.050596:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076888/real 1769076888] req@ffff9350100cb100 x1855008913370624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076888.050611:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913370624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076888.050622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076888.050625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076888.050629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913370624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076888.050636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913370752:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076888.050654:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076888.050664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076888/real 1769076888] req@ffff9350100c8000 x1855008913370752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076888.050679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913370752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076888.050689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076888.050692:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076888.050696:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913370752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076888.050702:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913370880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076888.050718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076888.050729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076888/real 1769076888] req@ffff9350100cad80 x1855008913370880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076888.050743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913370880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076888.050762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076888.050766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076888.050770:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913370880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076888.050776:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076888.050792:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076888.050803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076888/real 1769076888] req@ffff9350100c9f80 x1855008913371008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076888.050834:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913371008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076888.050844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076888.050847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076888.050850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076888.241234:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076888.753137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076888.753212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076888.753220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076888.753304:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076889.073225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076889.265113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076889.777183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076889.777187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076889.777198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076889.777232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076890.097189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076890.289157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076890.801173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076890.801228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076890.801251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076890.801289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076891.121220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076891.313114:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076891.825206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076891.825238:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076891.825246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076891.825286:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076892.145186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076892.337147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076892.849210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076892.849237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076892.849266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076892.849280:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076893.169219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076893.169230:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076893.169239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076893.169244:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076893.169267:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2598 00000100:00080000:0.0:1769076893.169281:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076893.169290:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076893.169350:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913371136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076893.169385:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076893.169390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076893.169392:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076893.169396:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2598 00000100:00080000:0.0:1769076893.169399:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076893.169403:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076893.169414:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076893.169426:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913371264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076893.169437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076893.169441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076893.169442:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076893.169446:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2598 00000100:00080000:0.0:1769076893.169449:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076893.169456:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076893.169456:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913371136:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769076893.169466:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076893.169476:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913371392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076893.169485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076893.169489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076893.169492:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076893.169495:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2598 00000100:00080000:0.0:1769076893.169499:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076893.169502:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076893.169507:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076893.169516:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913371520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076893.169524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076893.169528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076893.169530:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076893.169536:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2598 00000100:00080000:0.0:1769076893.169542:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076893.169548:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076893.169555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076893.169559:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076893.169578:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913371648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076893.169595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076893.169600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076893.169603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076893.169610:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2598 00000100:00080000:0.0:1769076893.169616:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076893.169622:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076893/real 1769076893] req@ffff9350103f9c00 x1855008913371136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076893.169625:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076893.169636:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076893.169654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913371136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076893.169657:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913371776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076893.169695:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076893.169696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076893.169705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076893.169709:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076893.169710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076893.169732:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2598 00000100:00100000:2.0:1769076893.169734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913371136:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076893.169737:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076893.169744:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076893.169748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913371264:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769076893.169751:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076893.169768:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913371904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076893.169782:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076893.169807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076893/real 1769076893] req@ffff9350103f9180 x1855008913371264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076893.169820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913371264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076893.169829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076893.169832:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076893.169835:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913371264:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076893.169862:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913371392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076893.169878:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076893.169887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076893/real 1769076893] req@ffff9350103fb100 x1855008913371392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076893.169897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913371392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076893.169929:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076893.169933:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076893.169936:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913371392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076893.169942:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913371520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076893.169962:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076893.169972:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076893/real 1769076893] req@ffff9350103f8380 x1855008913371520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076893.169982:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913371520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076893.169990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076893.169993:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076893.169996:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913371520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076893.170000:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371648:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076893.170013:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076893.170021:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076893/real 1769076893] req@ffff9350103f8700 x1855008913371648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076893.170030:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913371648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076893.170037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076893.170041:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076893.170043:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076893.170102:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076893.170121:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076893.170131:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076893/real 1769076893] req@ffff9350103faa00 x1855008913371776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076893.170142:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913371776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076893.170150:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076893.170153:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076893.170156:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076893.170161:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076893.170174:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076893.170182:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076893/real 1769076893] req@ffff9350103fb480 x1855008913371904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076893.170192:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913371904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076893.170198:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076893.170210:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076893.170213:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913371904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076893.361171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076893.873169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076893.873169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076893.873177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076893.873210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076894.193122:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076894.385149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076894.897135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076894.897135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076894.897135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076894.897169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076895.217109:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076895.409123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076895.921139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076895.921146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076895.921149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076895.921173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076896.241182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076896.433154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076896.945131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076896.945137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076896.945191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076896.945250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076897.265191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076897.457143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076897.969143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076897.969152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076897.969161:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076897.969181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076898.289152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076898.289165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076898.289171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076898.289175:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076898.289183:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2603 00000100:00080000:0.0:1769076898.289194:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076898.289207:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076898.289242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913372032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076898.289265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076898.289268:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076898.289270:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076898.289273:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2603 00000100:00080000:0.0:1769076898.289276:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076898.289278:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076898.289286:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076898.289294:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913372160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076898.289301:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076898.289303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076898.289304:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076898.289307:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2603 00000100:00080000:0.0:1769076898.289308:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076898.289311:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076898.289313:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076898.289320:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913372288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076898.289327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076898.289329:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076898.289331:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076898.289334:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2603 00000100:00080000:0.0:1769076898.289336:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076898.289338:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076898.289342:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076898.289352:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913372416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076898.289352:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913372032:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076898.289366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076898.289368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076898.289370:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076898.289373:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2603 00000100:00080000:0.0:1769076898.289375:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076898.289377:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076898.289380:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076898.289389:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913372544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076898.289396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076898.289398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076898.289399:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076898.289401:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2603 00000100:00080000:0.0:1769076898.289404:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076898.289406:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076898.289409:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076898.289428:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913372672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076898.289434:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076898.289440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076898.289444:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076898.289446:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076898.289449:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2603 00000100:00080000:0.0:1769076898.289452:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076898.289455:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076898.289459:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076898.289469:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913372800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076898.289475:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076898/real 1769076898] req@ffff9350103f9500 x1855008913372032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076898.289494:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913372032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076898.289514:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076898.289518:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076898.289521:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913372032:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076898.289526:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913372160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076898.289540:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076898.289548:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076898/real 1769076898] req@ffff9350103f9f80 x1855008913372160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076898.289557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913372160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076898.289563:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076898.289565:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076898.289567:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913372160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076898.289571:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913372288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076898.289580:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076898.289586:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076898/real 1769076898] req@ffff9350103fb800 x1855008913372288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076898.289593:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913372288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076898.289615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076898.289617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076898.289619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913372288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076898.289639:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913372416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076898.289659:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076898.289666:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076898/real 1769076898] req@ffff9350103fad80 x1855008913372416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076898.289674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913372416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076898.289680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076898.289683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076898.289685:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913372416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076898.289688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913372544:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076898.289697:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076898.289703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076898/real 1769076898] req@ffff9350103fbb80 x1855008913372544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076898.289710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913372544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076898.289716:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076898.289717:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076898.289720:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913372544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076898.289724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913372672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076898.289733:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076898.289738:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076898/real 1769076898] req@ffff9350103fb480 x1855008913372672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076898.289746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913372672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076898.289751:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076898.289753:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076898.289755:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913372672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076898.289758:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913372800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076898.289768:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076898.289774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076898/real 1769076898] req@ffff9350103faa00 x1855008913372800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076898.289781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913372800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076898.289787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076898.289788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076898.289790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913372800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076898.481154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076898.993197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076898.993227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076898.993298:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076898.993325:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076899.313180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076899.505155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076900.017189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076900.017233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076900.017265:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076900.017367:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076900.337161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076900.529155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076901.041186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076901.041186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076901.041312:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076901.041314:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076901.361189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076901.553105:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076902.065126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076902.065137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076902.065140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076902.065160:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076902.385195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076902.577208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076903.089141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076903.089141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076903.089141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076903.089165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076903.409125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076903.409138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076903.409156:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076903.409168:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2609 00000100:00080000:0.0:1769076903.409181:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076903.409190:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076903.409199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769076903.409247:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913372928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076903.409303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076903.409308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076903.409311:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076903.409317:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2609 00000100:00080000:0.0:1769076903.409322:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076903.409327:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076903.409349:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076903.409367:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913373056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076903.409380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076903.409385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076903.409388:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076903.409393:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2609 00000100:00080000:0.0:1769076903.409398:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076903.409403:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076903.409410:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076903.409442:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913373184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076903.409453:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913372928:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076903.409480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076903.409496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076903.409500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076903.409511:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2609 00000100:00080000:0.0:1769076903.409520:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076903.409529:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076903.409558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076903.409596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913373312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076903.409624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076903.409629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076903.409633:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076903.409635:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076903.409642:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2609 00000100:00080000:0.0:1769076903.409648:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076903.409653:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076903.409660:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076903.409673:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913373440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076903.409683:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076903.409686:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076903.409688:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076903.409692:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2609 00000100:00080000:0.0:1769076903.409696:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076903.409700:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076903.409705:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076903.409719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076903/real 1769076903] req@ffff9350103f8700 x1855008913372928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076903.409720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913373568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076903.409772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076903.409779:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076903.409781:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076903.409786:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2609 00000100:00080000:0.0:1769076903.409791:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076903.409799:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076903.409799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913372928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076903.409809:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076903.409828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913373696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076903.409829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076903.409845:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076903.409856:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913372928:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076903.409870:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373056:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076903.409911:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076903.409933:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076903/real 1769076903] req@ffff9350103f8380 x1855008913373056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076903.409954:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913373056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076903.409982:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076903.409988:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076903.409994:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076903.410029:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076903.410102:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076903.410120:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076903/real 1769076903] req@ffff9350103fb100 x1855008913373184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076903.410139:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913373184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076903.410162:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076903.410168:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076903.410173:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076903.410182:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076903.410205:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076903.410219:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076903/real 1769076903] req@ffff9350103f9180 x1855008913373312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076903.410237:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913373312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076903.410249:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076903.410254:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076903.410259:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076903.410267:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913373440:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076903.410305:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076903.410322:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076903/real 1769076903] req@ffff9350103f9c00 x1855008913373440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076903.410350:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913373440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076903.410376:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076903.410381:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076903.410386:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913373440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076903.410395:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913373568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076903.410422:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076903.410443:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076903/real 1769076903] req@ffff9350103faa00 x1855008913373568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076903.410462:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913373568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076903.410477:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076903.410482:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076903.410498:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913373568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076903.410507:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913373696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076903.410533:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076903.410549:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076903/real 1769076903] req@ffff9350103fb480 x1855008913373696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076903.410566:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913373696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076903.410595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076903.410600:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076903.410606:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913373696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076903.601164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076904.113157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076904.113158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076904.113187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076904.113188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076904.433227:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076904.625215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076905.137179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076905.137179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076905.137190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076905.137274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076905.457173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076905.649227:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076906.161145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076906.161173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076906.161185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076906.161268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076906.481197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076906.673163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076907.185118:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076907.185119:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076907.185166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076907.185219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076907.505175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076907.697166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076908.209134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076908.209144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076908.209196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076908.209241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076908.529173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076908.529181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076908.529190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076908.529194:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076908.529203:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2614 00000100:00080000:2.0:1769076908.529217:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076908.529225:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076908.529274:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913373824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076908.529306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076908.529311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076908.529313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076908.529319:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2614 00000100:00080000:2.0:1769076908.529325:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076908.529330:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076908.529341:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076908.529344:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913373824:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076908.529362:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913373952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076908.529375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076908.529378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076908.529380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076908.529383:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2614 00000100:00080000:2.0:1769076908.529387:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076908.529390:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076908.529395:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076908.529406:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913374080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076908.529411:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076908.529419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076908.529422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076908.529424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076908.529428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2614 00000100:00080000:2.0:1769076908.529434:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076908.529436:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076908.529439:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076908.529450:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913374208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076908.529456:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076908/real 1769076908] req@ffff935002a61c00 x1855008913373824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076908.529460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076908.529467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076908.529470:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076908.529477:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2614 00000100:00080000:2.0:1769076908.529481:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076908.529483:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913373824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076908.529489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076908.529497:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769076908.529499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076908.529503:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076908.529506:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913373824:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076908.529528:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076908.529531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913374336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076908.529547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076908.529549:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076908.529555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076908.529558:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076908.529562:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076908/real 1769076908] req@ffff935002a61f80 x1855008913373952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076908.529567:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2614 00000100:00080000:2.0:1769076908.529576:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076908.529580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913373952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076908.529586:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076908.529591:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076908.529596:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769076908.529596:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076908.529602:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913373952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076908.529609:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076908.529618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913374464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076908.529625:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076908.529635:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076908/real 1769076908] req@ffff935002a62680 x1855008913374080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076908.529635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076908.529660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076908.529666:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076908.529671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913374080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076908.529674:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2614 00000100:00080000:2.0:1769076908.529685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076908.529686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076908.529692:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076908.529696:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076908.529698:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076908.529707:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374208:192.168.204.121@tcp:8:kworker.0 00000008:00100000:2.0:1769076908.529708:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076908.529738:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913374592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076908.529739:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076908.529749:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076908/real 1769076908] req@ffff935002a60a80 x1855008913374208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076908.529760:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913374208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076908.529767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076908.529770:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076908.529772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076908.529787:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913374336:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076908.529798:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076908.529805:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076908/real 1769076908] req@ffff935002a63100 x1855008913374336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076908.529813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913374336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076908.529819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076908.529821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076908.529823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913374336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076908.529826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913374464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076908.529835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076908.529841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076908/real 1769076908] req@ffff935002a61180 x1855008913374464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076908.529848:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913374464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076908.529853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076908.529855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076908.529857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913374464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076908.529860:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913374592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076908.529869:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076908.529875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076908/real 1769076908] req@ffff935002a61880 x1855008913374592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076908.529882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913374592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076908.529892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076908.529894:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076908.529896:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913374592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076908.721133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076909.233242:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076909.233252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076909.233288:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076909.233311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076909.553125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076909.745127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076910.257179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076910.257222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076910.257255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076910.257284:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076910.577187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076910.769156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076911.281228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769076911.281228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076911.281231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076911.281352:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076911.601161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076911.793166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076912.305206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076912.305211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076912.305296:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076912.305341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076912.625164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076912.817125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076913.329155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076913.329169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076913.329181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076913.329245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076913.649114:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076913.649114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076913.649124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076913.649128:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076913.649136:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2619 00000100:00080000:2.0:1769076913.649145:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076913.649151:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076913.649192:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913374720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076913.649215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076913.649219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076913.649221:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076913.649225:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2619 00000100:00080000:2.0:1769076913.649229:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076913.649234:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076913.649244:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076913.649257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913374848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076913.649267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076913.649271:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913374720:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769076913.649271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076913.649273:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076913.649278:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2619 00000100:00080000:2.0:1769076913.649282:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076913.649285:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076913.649290:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076913.649299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913374976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076913.649308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076913.649311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076913.649312:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076913.649315:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2619 00000100:00080000:2.0:1769076913.649318:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076913.649321:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076913.649325:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076913.649333:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913375104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076913.649337:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076913.649342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076913.649345:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076913.649347:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076913.649351:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2619 00000100:00080000:2.0:1769076913.649357:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076913.649360:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076913.649364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076913.649376:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076913/real 1769076913] req@ffff935002a62a00 x1855008913374720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076913.649376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913375232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076913.649387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076913.649390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769076913.649393:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913374720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076913.649393:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076913.649396:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2619 00000100:00080000:2.0:1769076913.649400:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076913.649403:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076913.649404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076913.649408:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769076913.649408:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076913.649412:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913374720:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076913.649417:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913375360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076913.649425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076913.649428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076913.649430:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076913.649433:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2619 00000100:00100000:0.0:1769076913.649435:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374848:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076913.649437:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076913.649441:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076913.649445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076913.649455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913375488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076913.649456:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076913.649468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076913/real 1769076913] req@ffff935002a60000 x1855008913374848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076913.649480:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913374848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076913.649497:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076913.649500:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076913.649504:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076913.649509:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076913.649526:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076913.649534:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076913/real 1769076913] req@ffff935002a62300 x1855008913374976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076913.649542:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913374976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076913.649547:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076913.649549:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076913.649551:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913374976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076913.649554:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913375104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076913.649582:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076913.649589:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076913/real 1769076913] req@ffff935002a61500 x1855008913375104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076913.649597:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913375104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076913.649613:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076913.649615:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076913.649618:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913375104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076913.649621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913375232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076913.649632:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076913.649638:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076913/real 1769076913] req@ffff935002a61880 x1855008913375232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076913.649645:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913375232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076913.649650:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076913.649652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076913.649654:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913375232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076913.649657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913375360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076913.649666:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076913.649672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076913/real 1769076913] req@ffff935002a61180 x1855008913375360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076913.649679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913375360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076913.649684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076913.649686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076913.649687:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913375360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076913.649699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913375488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076913.649718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076913.649725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076913/real 1769076913] req@ffff935002a63100 x1855008913375488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076913.649732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913375488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076913.649737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076913.649739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076913.649741:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913375488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076913.841130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076914.353182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076914.353182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076914.353304:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076914.353333:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076914.673157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076914.865161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076915.377151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076915.377178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076915.377191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076915.377253:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076915.697202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076915.889146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076916.401149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076916.401149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076916.401175:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076916.401328:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076916.721138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076916.913127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076917.425146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076917.425176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076917.425177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076917.425208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076917.745192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076917.937140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076918.449226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076918.449226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076918.449231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076918.449267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076918.769157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076918.769188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076918.769196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076918.769202:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076918.769213:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2624 00000100:00080000:1.0:1769076918.769226:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076918.769235:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076918.769307:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913375616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076918.769333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076918.769337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076918.769338:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076918.769342:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2624 00000100:00080000:1.0:1769076918.769345:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076918.769348:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076918.769358:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076918.769369:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913375744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076918.769384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076918.769387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076918.769390:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076918.769393:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2624 00000100:00080000:1.0:1769076918.769413:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076918.769418:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076918.769424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076918.769437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913375872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076918.769449:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076918.769455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076918.769461:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076918.769467:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913375616:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076918.769468:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2624 00000100:00080000:1.0:1769076918.769504:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076918.769526:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076918.769532:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076918.769548:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913376000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076918.769557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076918.769559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076918.769561:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076918.769565:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2624 00000100:00080000:1.0:1769076918.769568:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076918.769571:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076918.769575:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076918.769584:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913376128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076918.769591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076918.769594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076918.769595:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076918.769598:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2624 00000100:00080000:1.0:1769076918.769604:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076918.769608:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076918.769612:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076918.769626:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913376256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076918.769637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076918.769640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076918.769642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076918.769647:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2624 00000100:00080000:1.0:1769076918.769652:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076918.769659:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076918.769668:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076918.769690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913376384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076918.769706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913375744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076918.769775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913375872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076918.769824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076918.769862:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913376128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076918.769888:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913376256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076918.769918:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913376384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076918.961172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076919.473154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076919.473154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076919.473154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076919.473188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076919.793396:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076919.985190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076920.497160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076920.497161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076920.497163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076920.497195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076920.817282:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076921.009169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076921.521143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076921.521143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076921.521209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076921.521220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076921.841149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769076921.841330:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769076921.841373:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076921.841392:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076921.841454:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076921.841467:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076921.841478:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076921.841490:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076921.841502:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076918/real 1769076921] req@ffff9350100ca680 x1855008913375872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769076921.841503:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769076921.841517:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076921.841534:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913375872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076921.841546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076921.841552:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076921.841557:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913375872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076921.841565:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076918/real 1769076921] req@ffff9350100c8e00 x1855008913376000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076921.841579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913376000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076921.841587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076921.841590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076921.841594:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076921.841601:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076918/real 1769076921] req@ffff9350100c9880 x1855008913376128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076921.841614:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913376128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076921.841622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076921.841626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076921.841630:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913376128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076921.841637:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076918/real 1769076921] req@ffff9350100c9500 x1855008913376256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076921.841649:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913376256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076921.841657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076921.841661:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076921.841664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913376256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076921.841670:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076918/real 1769076921] req@ffff9350100caa00 x1855008913376384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076921.841680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913376384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076921.841688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076921.841692:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076921.841695:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913376384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076921.841701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076918/real 1769076921] req@ffff9350100c8380 x1855008913375616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076921.841714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913375616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076921.841721:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076921.841724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076921.841727:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913375616:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076921.841734:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076918/real 1769076921] req@ffff9350100ca300 x1855008913375744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076921.841746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913375744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076921.841756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076921.841759:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076921.841762:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913375744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076922.033158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076922.545166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076922.545184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076922.545188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076922.545204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076922.865118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076923.057162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076923.569150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076923.569171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076923.569200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076923.569225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076923.889136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076923.889191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.889198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076923.889202:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.889209:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2629 00000100:00080000:3.0:1769076923.889219:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.889225:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076923.889255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913376512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076923.889286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.889289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076923.889290:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.889294:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2629 00000100:00080000:3.0:1769076923.889296:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.889298:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076923.889306:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076923.889315:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913376640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076923.889334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.889337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076923.889339:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.889342:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2629 00000100:00080000:3.0:1769076923.889344:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.889347:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076923.889351:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076923.889359:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913376512:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076923.889361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913376768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076923.889373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.889377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076923.889379:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.889384:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2629 00000100:00080000:3.0:1769076923.889418:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.889424:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076923.889430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076923.889455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913376896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076923.889468:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076923.889471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.889476:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076923.889479:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.889490:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2629 00000100:00080000:3.0:1769076923.889496:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.889501:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076923.889518:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076923.889528:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff935002a7fb80 x1855008913376512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076923.889538:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913377024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076923.889553:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076923.889555:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913376512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076923.889560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076923.889565:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.889576:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2629 00000100:00080000:0.0:1769076923.889582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076923.889583:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076923.889593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076923.889595:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076923.889603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913376512:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076923.889606:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076923.889615:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076923.889640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913377152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076923.889651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076923.889667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076923.889672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff935002a7dc00 x1855008913376640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076923.889674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076923.889687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.889698:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2629 00000100:00100000:0.0:1769076923.889708:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913376640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076923.889708:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.889720:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076923.889731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076923.889733:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076923.889740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076923.889747:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769076923.889753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913377280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076923.889788:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076923.889816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076923.889831:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff935002a7d180 x1855008913376768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.889846:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913376768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.889855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.889859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076923.889863:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076923.889872:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076923.889895:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076923.889908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff935002a7df80 x1855008913376896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.889931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913376896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.889942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.889945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076923.889950:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913376896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076923.889957:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377024:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076923.889982:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076923.889995:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff935002a7c000 x1855008913377024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.890010:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913377024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.890020:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.890024:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076923.890028:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076923.890033:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076923.890087:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076923.890101:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff935002a7f800 x1855008913377152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.890115:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913377152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.890124:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.890128:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076923.890131:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076923.890138:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076923.890161:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076923.890179:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff935002a7c700 x1855008913377280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.890193:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913377280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.890202:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.890208:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076923.890212:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076924.081133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076924.593128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076924.593132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076924.593157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076924.593187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076924.913192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076925.105177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076925.617137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076925.617169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076925.617170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076925.617180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076925.937329:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076926.129144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076926.641250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076926.641265:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076926.641266:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076926.641377:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076926.961231:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076927.153236:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076927.665184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076927.665241:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076927.665251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076927.665254:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076927.985124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076928.177116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076928.689124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076928.689128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076928.689177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076928.689268:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076929.009218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076929.009268:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076929.009279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076929.009285:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076929.009297:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2634 00000100:00080000:0.0:1769076929.009374:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076929.009383:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076929.009437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913377408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076929.009475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076929.009479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076929.009481:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076929.009486:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2634 00000100:00080000:0.0:1769076929.009489:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076929.009494:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076929.009518:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076929.009530:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913377536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076929.009540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076929.009544:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076929.009546:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076929.009550:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2634 00000100:00080000:0.0:1769076929.009553:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076929.009556:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076929.009561:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076929.009570:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913377664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076929.009587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913377408:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076929.009589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076929.009595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076929.009597:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076929.009601:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2634 00000100:00080000:0.0:1769076929.009607:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076929.009616:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076929.009620:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076929.009631:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913377792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076929.009639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076929.009645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076929.009647:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076929.009651:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2634 00000100:00080000:0.0:1769076929.009655:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076929.009660:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076929.009663:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076929.009673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913377920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076929.009682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076929.009685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076929.009687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076929.009695:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2634 00000800:00000100:2.0:1769076929.009696:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076929.009712:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076929.009721:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076929.009727:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076929.009739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913378048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076929.009750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076929.009754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076929.009756:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076929.009762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076929/real 1769076929] req@ffff9350103fbb80 x1855008913377408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076929.009765:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2634 00000100:00080000:0.0:1769076929.009772:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076929.009788:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076929.009799:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076929.009806:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913377408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076929.009814:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913378176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076929.009832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076929.009843:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076929.009850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913377408:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076929.009859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913377536:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076929.009880:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076929.009888:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076929/real 1769076929] req@ffff9350103fad80 x1855008913377536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076929.009897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913377536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076929.009904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076929.009907:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076929.009909:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913377536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076929.009931:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913377664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076929.009943:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076929.009959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076929/real 1769076929] req@ffff9350103fb800 x1855008913377664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076929.009967:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913377664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076929.009973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076929.009975:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076929.009977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913377664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076929.009981:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913377792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076929.009990:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076929.009995:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076929/real 1769076929] req@ffff9350103f9f80 x1855008913377792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076929.010003:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913377792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076929.010008:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076929.010010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076929.010012:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913377792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076929.010016:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377920:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076929.010027:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076929.010032:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076929/real 1769076929] req@ffff9350103f9500 x1855008913377920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076929.010040:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913377920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076929.010045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076929.010075:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076929.010077:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913377920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076929.010081:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076929.010094:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076929.010101:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076929/real 1769076929] req@ffff9350103fb480 x1855008913378048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076929.010109:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913378048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076929.010115:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076929.010117:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076929.010119:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076929.010123:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076929.010132:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076929.010138:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076929/real 1769076929] req@ffff9350103faa00 x1855008913378176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076929.010150:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913378176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076929.010155:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076929.010157:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076929.010158:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076929.201168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076929.713228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076929.713287:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076929.713315:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076929.713316:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076930.033140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076930.225111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076930.737144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076930.737179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076930.737221:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076930.737247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076931.057195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076931.249224:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076931.761189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076931.761190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076931.761229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076931.761235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076932.081146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076932.273236:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076932.785148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076932.785170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076932.785173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076932.785229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076933.105220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076933.297166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076933.809210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076933.809211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076933.809211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076933.809241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076934.129158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076934.129162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076934.129172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076934.129178:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076934.129191:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2639 00000100:00080000:0.0:1769076934.129205:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076934.129252:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076934.129299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913378304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076934.129326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076934.129329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076934.129331:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076934.129334:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2639 00000100:00080000:0.0:1769076934.129337:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076934.129340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076934.129352:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076934.129363:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913378432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076934.129378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076934.129381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076934.129383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076934.129389:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913378304:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076934.129390:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2639 00000100:00080000:0.0:1769076934.129397:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076934.129409:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076934.129412:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076934.129426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913378560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076934.129434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076934.129437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076934.129438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076934.129441:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2639 00000100:00080000:0.0:1769076934.129443:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076934.129446:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076934.129449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076934.129455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913378688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076934.129462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076934.129465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076934.129466:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076934.129472:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2639 00000800:00000100:2.0:1769076934.129481:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076934.129483:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076934.129489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076934.129493:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076934.129504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913378816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076934.129512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076934.129514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076934.129517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076934.129520:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2639 00000100:00080000:0.0:1769076934.129524:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076934.129528:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076934.129531:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076934.129536:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076934/real 1769076934] req@ffff9350103f9c00 x1855008913378304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076934.129545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913378944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076934.129575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076934.129578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076934.129580:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076934.129585:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913378304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076934.129589:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2639 00000100:00080000:0.0:1769076934.129602:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076934.129616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076934.129617:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076934.129631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:0.0:1769076934.129635:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076934.129636:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913378304:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076934.129667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913379072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076934.129673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913378432:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076934.129716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076934.129728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076934/real 1769076934] req@ffff9350103f9180 x1855008913378432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076934.129738:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913378432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076934.129744:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076934.129747:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076934.129749:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913378432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076934.129753:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913378560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076934.129765:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076934.129770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076934/real 1769076934] req@ffff9350103fb100 x1855008913378560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076934.129778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913378560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076934.129783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076934.129785:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076934.129787:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913378560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076934.129791:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913378688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076934.129800:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076934.129805:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076934/real 1769076934] req@ffff9350103f8380 x1855008913378688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076934.129812:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913378688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076934.129821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076934.129823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076934.129825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913378688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076934.129829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378816:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076934.129838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076934.129844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076934/real 1769076934] req@ffff9350103f8700 x1855008913378816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076934.129850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913378816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076934.129855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076934.129857:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076934.129859:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076934.129863:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076934.129872:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076934.129878:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076934/real 1769076934] req@ffff9350103faa00 x1855008913378944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076934.129884:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913378944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076934.129889:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076934.129891:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076934.129894:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913378944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076934.129908:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076934.129918:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076934.129923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076934/real 1769076934] req@ffff9350103fb480 x1855008913379072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076934.129931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913379072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076934.129935:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076934.129937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076934.129939:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076934.321129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076934.833124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076934.833130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076934.833194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076934.833267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076935.153296:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076935.345318:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076935.857178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076935.857179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076935.857255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076935.857296:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076936.177263:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076936.369171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076936.881142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076936.881143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076936.881145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076936.881234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076937.201224:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076937.393214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076937.905227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076937.905228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076937.905261:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076937.905279:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076938.225171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076938.417155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076938.929130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076938.929131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076938.929157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076938.929212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076939.249218:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076939.249223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076939.249232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076939.249237:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076939.249247:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2644 00000100:00080000:0.0:1769076939.249258:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076939.249265:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076939.249319:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913379200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076939.249349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076939.249353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076939.249355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076939.249359:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2644 00000100:00080000:0.0:1769076939.249362:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076939.249366:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076939.249386:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076939.249407:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913379200:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076939.249408:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913379328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076939.249428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076939.249432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076939.249435:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076939.249439:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2644 00000100:00080000:0.0:1769076939.249443:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076939.249446:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076939.249452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076939.249479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913379456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076939.249486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076939.249489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076939.249491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076939.249493:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2644 00000100:00080000:0.0:1769076939.249497:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076939.249502:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076939.249508:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076939.249514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076939.249525:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913379584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076939.249534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076939.249536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076939.249538:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076939.249542:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2644 00000100:00100000:2.0:1769076939.249547:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076939/real 1769076939] req@ffff9350103f9500 x1855008913379200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076939.249548:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076939.249558:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076939.249586:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076939.249595:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913379200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076939.249602:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913379712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076939.249613:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076939.249619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076939.249624:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076939.249631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769076939.249633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913379200:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076939.249651:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076939.249662:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2644 00000100:00080000:0.0:1769076939.249668:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076939.249674:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076939.249679:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913379328:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769076939.249701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076939.249733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913379840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076939.249734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076939.249750:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076939/real 1769076939] req@ffff9350103f9f80 x1855008913379328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076939.249753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076939.249766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076939.249770:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076939.249773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913379328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076939.249780:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2644 00000100:00080000:0.0:1769076939.249798:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076939.249800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076939.249806:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076939.249812:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076939.249816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076939.249838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913379328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076939.249849:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913379968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076939.249854:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913379456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076939.249894:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076939.249908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076939/real 1769076939] req@ffff9350103fb800 x1855008913379456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076939.249920:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913379456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076939.249945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076939.249948:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076939.249951:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913379456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076939.249955:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913379584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076939.249967:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076939.249983:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076939/real 1769076939] req@ffff9350103fad80 x1855008913379584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076939.249992:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913379584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076939.249998:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076939.250000:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076939.250002:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913379584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076939.250007:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379712:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076939.250018:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076939.250043:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076939/real 1769076939] req@ffff9350103fbb80 x1855008913379712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076939.250063:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913379712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076939.250071:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076939.250073:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076939.250076:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076939.250164:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076939.250197:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076939.250209:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076939/real 1769076939] req@ffff9350103f9500 x1855008913379840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076939.250219:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913379840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076939.250231:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076939.250234:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076939.250236:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076939.250240:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076939.250254:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076939.250261:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076939/real 1769076939] req@ffff9350103fb480 x1855008913379968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076939.250269:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913379968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076939.250286:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076939.250288:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076939.250290:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913379968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076939.441166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076939.953177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076939.953196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076939.953222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076939.953225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076940.273173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076940.465212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076940.977150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076940.977190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076940.977197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076940.977219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076941.297177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076941.489170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076942.001178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076942.001178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076942.001178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076942.001206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076942.321199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076942.513174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076943.025223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076943.025225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076943.025234:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076943.025235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076943.345197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076943.537202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076944.049179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076944.049179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076944.049188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076944.049361:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076944.369233:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076944.369238:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076944.369246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076944.369252:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076944.369264:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2650 00000100:00080000:2.0:1769076944.369277:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076944.369286:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076944.369360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913380096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076944.369402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076944.369406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076944.369409:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076944.369415:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2650 00000100:00080000:2.0:1769076944.369421:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076944.369427:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076944.369440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076944.369459:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913380096:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076944.369461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913380224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076944.369484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076944.369488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076944.369491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076944.369495:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2650 00000100:00080000:2.0:1769076944.369500:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076944.369503:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076944.369509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076944.369545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913380352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076944.369558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076944.369563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076944.369575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076944.369576:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076944.369588:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2650 00000100:00080000:2.0:1769076944.369596:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076944.369602:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076944.369611:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076944.369625:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076944/real 1769076944] req@ffff935002a60a80 x1855008913380096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076944.369632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913380480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076944.369657:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076944.369662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769076944.369663:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913380096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076944.369665:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076944.369676:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2650 00000100:00080000:2.0:1769076944.369684:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076944.369692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076944.369693:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076944.369705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769076944.369705:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076944.369713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913380096:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076944.369724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913380608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076944.369735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076944.369738:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076944.369741:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076944.369742:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913380224:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076944.369748:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2650 00000100:00080000:2.0:1769076944.369753:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076944.369756:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076944.369760:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076944.369775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913380736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076944.369780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769076944.369795:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076944.369800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076944/real 1769076944] req@ffff935002a62680 x1855008913380224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076944.369801:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076944.369805:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076944.369815:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2650 00000100:00080000:2.0:1769076944.369822:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076944.369827:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913380224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076944.369829:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076944.369838:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076944.369844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076944.369851:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076944.369856:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913380864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076944.369861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913380224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076944.369876:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913380352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076944.369898:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076944.369908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076944/real 1769076944] req@ffff935002a61f80 x1855008913380352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076944.369936:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913380352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076944.369944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076944.369947:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076944.369950:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913380352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076944.369955:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913380480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076944.369970:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076944.369978:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076944/real 1769076944] req@ffff935002a61c00 x1855008913380480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076944.369998:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913380480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076944.370006:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076944.370008:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076944.370010:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913380480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076944.370015:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913380608:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076944.370028:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076944.370035:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076944/real 1769076944] req@ffff935002a63100 x1855008913380608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076944.370136:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913380608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076944.370154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076944.370158:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076944.370161:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913380608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076944.370184:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913380736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076944.370210:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076944.370238:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076944/real 1769076944] req@ffff935002a61180 x1855008913380736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076944.370248:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913380736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076944.370255:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076944.370257:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076944.370260:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913380736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076944.370264:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913380864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076944.370277:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076944.370285:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076944/real 1769076944] req@ffff935002a61880 x1855008913380864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076944.370313:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913380864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076944.370332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076944.370335:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076944.370339:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913380864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076944.561184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076945.073142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076945.073172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076945.073177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076945.073256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076945.393146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076945.585174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076946.097238:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076946.097255:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076946.097368:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076946.097406:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076946.417285:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076946.609162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076947.121201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076947.121218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076947.121273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076947.121313:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076947.441147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076947.633159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076948.145160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076948.145177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076948.145213:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076948.145224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076948.465192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076948.657160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076949.169159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076949.169159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076949.169159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076949.169233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076949.489188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076949.489203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769076949.489210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076949.489217:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076949.489228:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2655 00000100:00080000:2.0:1769076949.489240:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076949.489248:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076949.489295:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913380992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076949.489333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076949.489339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076949.489342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076949.489348:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2655 00000100:00080000:2.0:1769076949.489355:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076949.489362:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076949.489376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076949.489390:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913380992:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769076949.489396:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913381120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076949.489410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076949.489413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076949.489416:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076949.489420:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2655 00000100:00080000:2.0:1769076949.489424:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076949.489428:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076949.489433:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769076949.489444:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913381248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076949.489453:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076949.489456:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076949.489458:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076949.489465:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2655 00000100:00080000:2.0:1769076949.489469:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076949.489471:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076949.489475:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076949.489477:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076949.489487:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913381376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076949.489505:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076949.489508:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076949.489509:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076949.489513:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2655 00000100:00080000:2.0:1769076949.489517:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076949.489521:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076949.489526:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076949.489528:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076949/real 1769076949] req@ffff935002a61500 x1855008913380992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076949.489542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913381504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076949.489553:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913380992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076949.489555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076949.489566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076949.489571:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076949.489583:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2655 00000100:00080000:0.0:1769076949.489584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076949.489588:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076949.489594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769076949.489598:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076949.489604:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913380992:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769076949.489606:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076949.489623:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913381632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076949.489634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076949.489638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913381120:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769076949.489639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076949.489645:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076949.489651:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2655 00000100:00080000:2.0:1769076949.489655:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076949.489659:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769076949.489664:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076949.489670:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076949.489679:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913381760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076949.489689:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076949/real 1769076949] req@ffff935002a62300 x1855008913381120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076949.489705:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913381120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076949.489714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076949.489718:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076949.489721:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913381120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076949.489728:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913381248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076949.489744:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076949.489752:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076949/real 1769076949] req@ffff935002a60000 x1855008913381248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076949.489763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913381248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076949.489770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076949.489772:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076949.489775:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913381248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076949.489780:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913381376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076949.489793:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076949.489800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076949/real 1769076949] req@ffff935002a62a00 x1855008913381376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076949.489809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913381376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076949.489816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076949.489819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076949.489821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913381376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076949.489826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913381504:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076949.489838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076949.489845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076949/real 1769076949] req@ffff935002a61880 x1855008913381504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076949.489855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913381504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076949.489863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076949.489865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076949.489868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913381504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076949.489883:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913381632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076949.489896:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076949.489903:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076949/real 1769076949] req@ffff935002a61180 x1855008913381632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076949.489912:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913381632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076949.489919:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076949.489922:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076949.489925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913381632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076949.489930:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913381760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076949.489942:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076949.489950:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076949/real 1769076949] req@ffff935002a63100 x1855008913381760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076949.490012:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913381760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076949.490028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076949.490032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076949.490035:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913381760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076949.681178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076950.193129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076950.193138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076950.193144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076950.193216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076950.513154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076950.705149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076951.217229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076951.217234:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076951.217250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076951.217323:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076951.537171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076951.729152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076952.241126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076952.241129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076952.241130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076952.241171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076952.561179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076952.753158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076953.265161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076953.265165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076953.265165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076953.265220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076953.585158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076953.777144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076954.289135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076954.289137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076954.289168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076954.289202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076954.609127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076954.609155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076954.609163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076954.609167:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076954.609177:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2660 00000100:00080000:3.0:1769076954.609190:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076954.609197:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076954.609231:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913381888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076954.609258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076954.609262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076954.609264:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076954.609269:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2660 00000100:00080000:3.0:1769076954.609273:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076954.609278:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076954.609288:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076954.609301:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913382016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076954.609312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076954.609315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076954.609316:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076954.609320:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2660 00000100:00080000:3.0:1769076954.609324:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076954.609326:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076954.609331:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076954.609338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913381888:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769076954.609341:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913382144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076954.609353:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076954.609355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076954.609357:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076954.609359:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2660 00000100:00080000:3.0:1769076954.609362:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076954.609364:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076954.609367:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076954.609375:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913382272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076954.609381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076954.609384:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076954.609385:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076954.609387:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2660 00000100:00080000:3.0:1769076954.609390:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076954.609393:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076954.609396:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076954.609404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913382400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076954.609409:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769076954.609416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076954.609419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076954.609420:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076954.609424:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2660 00000100:00080000:3.0:1769076954.609427:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076954.609430:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076954.609433:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076954.609441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913382528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076954.609448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076954.609451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076954.609452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076954/real 1769076954] req@ffff935002a7f100 x1855008913381888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076954.609453:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076954.609458:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2660 00000100:00080000:3.0:1769076954.609460:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076954.609462:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076954.609465:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076954.609476:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913382656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076954.609485:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913381888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076954.609495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076954.609499:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076954.609502:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913381888:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076954.609508:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382016:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076954.609524:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076954.609533:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076954/real 1769076954] req@ffff935002a7c380 x1855008913382016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076954.609546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913382016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076954.609552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076954.609554:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076954.609556:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076954.609576:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076954.609587:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076954.609593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076954/real 1769076954] req@ffff935002a7ca80 x1855008913382144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076954.609600:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913382144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076954.609605:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076954.609608:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076954.609610:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076954.609613:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076954.609622:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076954.609634:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076954/real 1769076954] req@ffff935002a7e300 x1855008913382272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076954.609642:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913382272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076954.609651:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076954.609653:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076954.609656:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076954.609659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913382400:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076954.609669:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076954.609675:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076954/real 1769076954] req@ffff935002a7c700 x1855008913382400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076954.609682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913382400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076954.609687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076954.609689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076954.609692:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913382400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076954.609695:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913382528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076954.609712:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076954.609718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076954/real 1769076954] req@ffff935002a7f800 x1855008913382528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076954.609725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913382528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076954.609730:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076954.609732:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076954.609735:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913382528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076954.609738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913382656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076954.609759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076954.609764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076954/real 1769076954] req@ffff935002a7c000 x1855008913382656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076954.609771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913382656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076954.609776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076954.609778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076954.609780:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913382656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076954.801114:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076955.313131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076955.313131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076955.313132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076955.313163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076955.633136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076955.825183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076956.337123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076956.337124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076956.337139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076956.337162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076956.657143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076956.849136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076957.361147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076957.361149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076957.361150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076957.361213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076957.681130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076957.873152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076958.385148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076958.385157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076958.385159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076958.385180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076958.705170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076958.897136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076959.409138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076959.409139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076959.409139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076959.409172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076959.729203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076959.729231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076959.729242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076959.729248:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076959.729261:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2665 00000100:00080000:0.0:1769076959.729277:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076959.729287:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076959.729332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913382784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076959.729367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076959.729372:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076959.729374:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076959.729379:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2665 00000100:00080000:0.0:1769076959.729383:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076959.729387:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076959.729400:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076959.729441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913382912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076959.729454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076959.729456:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913382784:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076959.729462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076959.729470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076959.729477:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2665 00000100:00080000:0.0:1769076959.729482:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076959.729488:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076959.729496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076959.729517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913383040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076959.729530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076959.729538:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076959.729539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076959.729545:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076959.729554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2665 00000100:00080000:0.0:1769076959.729560:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076959.729572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076959.729579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076959.729596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913383168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076959.729600:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076959/real 1769076959] req@ffff9350103faa00 x1855008913382784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076959.729623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076959.729628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076959.729630:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076959.729636:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913382784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076959.729637:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2665 00000100:00080000:0.0:1769076959.729645:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076959.729653:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076959.729661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076959.729662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076959.729671:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076959.729678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913383296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076959.729679:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913382784:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769076959.729707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076959.729710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076959.729712:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076959.729716:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2665 00000100:00080000:0.0:1769076959.729719:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076959.729723:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076959.729727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382912:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769076959.729731:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076959.729748:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913383424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076959.729759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076959.729760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769076959.729767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076959.729770:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076959.729775:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2665 00000100:00080000:0.0:1769076959.729780:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076959.729781:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076959/real 1769076959] req@ffff9350103f8700 x1855008913382912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076959.729809:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076959.729817:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076959.729845:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913382912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076959.729848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913383552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076959.729870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076959.729877:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076959.729881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913382912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076959.729889:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076959.729911:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076959.729922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076959/real 1769076959] req@ffff9350103f8380 x1855008913383040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076959.729934:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913383040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076959.729941:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076959.729945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076959.729948:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076959.729952:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076959.729966:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076959.729973:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076959/real 1769076959] req@ffff9350103fb100 x1855008913383168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076959.729983:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913383168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076959.729991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076959.729993:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076959.729996:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076959.730000:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913383296:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076959.730014:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076959.730022:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076959/real 1769076959] req@ffff9350103f9180 x1855008913383296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076959.730033:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913383296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076959.730040:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076959.730042:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076959.730045:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913383296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076959.730099:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913383424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076959.730128:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076959.730139:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076959/real 1769076959] req@ffff9350103f9c00 x1855008913383424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076959.730150:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913383424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076959.730158:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076959.730161:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076959.730164:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913383424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076959.730169:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913383552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076959.730196:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076959.730227:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076959/real 1769076959] req@ffff9350103faa00 x1855008913383552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076959.730261:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913383552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076959.730284:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076959.730290:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076959.730295:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913383552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076959.921183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076960.433139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076960.433142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076960.433150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076960.433238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076960.753162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076960.945133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076961.457210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076961.457211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076961.457217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076961.457270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076961.777153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076961.969123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076962.481128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076962.481128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076962.481134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076962.481170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076962.801145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076962.993156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076963.505161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076963.505161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076963.505261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076963.505275:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076963.825154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076964.017172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076964.529165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076964.529165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076964.529181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076964.529261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076964.849131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769076964.849180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076964.849188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076964.849192:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076964.849203:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2670 00000100:00080000:3.0:1769076964.849215:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076964.849222:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769076964.849268:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913383680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076964.849297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076964.849300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076964.849301:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076964.849305:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2670 00000100:00080000:3.0:1769076964.849307:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076964.849309:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076964.849320:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076964.849334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913383808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076964.849346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076964.849350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076964.849354:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076964.849359:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2670 00000100:00080000:3.0:1769076964.849364:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076964.849371:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076964.849380:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913383680:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769076964.849382:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076964.849412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913383936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076964.849430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076964.849435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076964.849438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076964.849445:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2670 00000100:00080000:3.0:1769076964.849469:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076964.849473:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076964.849480:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769076964.849482:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769076964.849502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913384064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076964.849512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076964.849515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076964.849516:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076964.849520:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2670 00000100:00080000:3.0:1769076964.849524:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076964.849528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076964.849533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076964.849549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913384192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076964.849553:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076964/real 1769076964] req@ffff935002a7df80 x1855008913383680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076964.849581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076964.849595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076964.849603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076964.849617:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2670 00000100:00080000:3.0:1769076964.849644:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076964.849651:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076964.849659:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076964.849667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913383680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076964.849679:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913384320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076964.849733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076964.849735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076964.849745:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769076964.849761:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076964.849762:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913383680:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769076964.849768:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076964.849774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383808:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769076964.849779:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2670 00000100:00080000:3.0:1769076964.849789:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076964.849796:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769076964.849804:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076964.849812:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769076964.849826:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913384448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076964.849834:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076964/real 1769076964] req@ffff935002a7d180 x1855008913383808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076964.849880:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913383808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076964.849890:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076964.849894:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076964.849897:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076964.849922:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076964.849937:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076964.849944:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076964/real 1769076964] req@ffff935002a7dc00 x1855008913383936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076964.849963:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913383936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076964.849970:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076964.849990:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076964.849994:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913383936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076964.850001:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076964.850022:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076964.850033:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076964/real 1769076964] req@ffff935002a7fb80 x1855008913384064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076964.850046:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913384064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076964.850055:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076964.850058:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076964.850062:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076964.850124:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913384192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076964.850146:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076964.850158:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076964/real 1769076964] req@ffff935002a7c000 x1855008913384192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076964.850172:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913384192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076964.850181:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076964.850185:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076964.850189:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913384192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076964.850195:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913384320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076964.850212:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076964.850221:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076964/real 1769076964] req@ffff935002a7f800 x1855008913384320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076964.850233:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913384320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076964.850241:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076964.850244:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076964.850248:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913384320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076964.850253:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913384448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076964.850269:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076964.850278:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076964/real 1769076964] req@ffff935002a7c700 x1855008913384448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076964.850290:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913384448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076964.850299:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076964.850302:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076964.850305:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913384448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076965.041134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076965.553180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076965.553182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076965.553323:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076965.553324:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076965.873179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076966.065203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076966.577255:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076966.577275:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076966.577308:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076966.577311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076966.897138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076967.089179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076967.601115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076967.601127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076967.601135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076967.601170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076967.921159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076968.113152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076968.625149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076968.625149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076968.625161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076968.625182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076968.945150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076969.137167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076969.649193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076969.649215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076969.649259:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076969.649281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076969.969169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076969.969171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076969.969185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076969.969189:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076969.969199:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2675 00000100:00080000:1.0:1769076969.969211:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076969.969218:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076969.969254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913384576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076969.969278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076969.969281:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076969.969282:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076969.969286:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2675 00000100:00080000:1.0:1769076969.969288:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076969.969291:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076969.969298:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076969.969310:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913384704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076969.969319:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076969.969322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076969.969323:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076969.969328:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2675 00000100:00100000:2.0:1769076969.969330:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913384576:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076969.969331:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076969.969336:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076969.969345:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076969.969354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913384832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076969.969361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076969.969364:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076969.969365:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076969.969368:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2675 00000100:00080000:1.0:1769076969.969370:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076969.969373:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076969.969376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076969.969383:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913384960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076969.969390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076969.969392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076969.969394:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076969.969397:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2675 00000100:00080000:1.0:1769076969.969400:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076969.969402:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076969.969405:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076969.969412:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076969.969416:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913385088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076969.969424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076969.969427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076969.969430:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076969.969434:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2675 00000100:00080000:1.0:1769076969.969437:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076969.969441:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076969.969446:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076969.969457:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9350100cb800 x1855008913384576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076969.969462:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913385216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076969.969483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076969.969487:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913384576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076969.969490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076969.969495:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076969.969503:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076969.969506:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2675 00000100:00080000:2.0:1769076969.969511:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076969.969515:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076969.969516:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913384576:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076969.969525:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076969.969531:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076969.969566:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076969.969584:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913385344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076969.969673:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076969.969689:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9350100ca300 x1855008913384704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.969706:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913384704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.969713:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.969716:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076969.969719:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076969.969724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076969.969739:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076969.969746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9350100c8380 x1855008913384832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.969755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913384832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.969761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.969763:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076969.969766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076969.969770:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076969.969780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076969.969785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9350100caa00 x1855008913384960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.969793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913384960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.969804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.969807:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076969.969809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913384960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076969.969813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385088:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076969.969823:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076969.969829:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9350100c9500 x1855008913385088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.969836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913385088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.969850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.969853:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076969.969855:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076969.969859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076969.969872:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076969.969879:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9350100c9880 x1855008913385216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.969887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913385216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.969892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.969894:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076969.969897:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076969.969911:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076969.969928:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076969.969936:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9350100c8e00 x1855008913385344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.969944:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913385344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.969950:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.969953:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076969.969955:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076970.161172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076970.673163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076970.673163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076970.673163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076970.673205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076970.993156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076971.185120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076971.697141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076971.697141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076971.697154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769076971.697197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076972.017137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076972.209143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076972.721165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076972.721184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076972.721189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076972.721272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076973.041153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076973.233137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076973.745136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076973.745154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076973.745175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769076973.745253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076974.065198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076974.257160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076974.769195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076974.769212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076974.769217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076974.769273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076975.089155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769076975.089163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076975.089173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076975.089179:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076975.089194:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2680 00000100:00080000:0.0:1769076975.089223:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076975.089232:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076975.089287:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913385472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076975.089318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076975.089321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076975.089323:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076975.089328:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2680 00000100:00080000:0.0:1769076975.089332:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076975.089337:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076975.089347:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076975.089360:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913385472:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076975.089364:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913385600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076975.089379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076975.089382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076975.089384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076975.089388:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2680 00000100:00080000:0.0:1769076975.089391:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076975.089394:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076975.089399:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076975.089410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913385728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076975.089418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076975.089421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076975.089424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076975.089428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2680 00000100:00080000:0.0:1769076975.089432:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076975.089435:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769076975.089441:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:0.0:1769076975.089442:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076975.089481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913385856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076975.089489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076975.089491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076975.089493:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076975.089497:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2680 00000100:00080000:0.0:1769076975.089501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076975.089504:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076975/real 1769076975] req@ffff9350103fb480 x1855008913385472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076975.089507:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076975.089515:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076975.089541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913385984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076975.089543:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913385472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076975.089561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076975.089582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076975.089584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076975.089585:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076975.089596:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2680 00000100:00080000:0.0:1769076975.089602:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076975.089602:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769076975.089610:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076975.089617:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913385472:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769076975.089618:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076975.089639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913386112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076975.089648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076975.089652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076975.089654:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076975.089662:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2680 00000100:00100000:2.0:1769076975.089664:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913385600:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769076975.089665:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076975.089672:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769076975.089689:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076975.089703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913386240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076975.089713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076975.089740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076975/real 1769076975] req@ffff9350103f9500 x1855008913385600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076975.089755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913385600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076975.089764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076975.089767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076975.089771:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913385600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076975.089777:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913385728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076975.089793:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076975.089802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076975/real 1769076975] req@ffff9350103fbb80 x1855008913385728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076975.089813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913385728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076975.089820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076975.089823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076975.089827:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913385728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076975.089831:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913385856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076975.089844:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076975.089866:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076975/real 1769076975] req@ffff9350103fad80 x1855008913385856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076975.089873:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913385856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076975.089879:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076975.089882:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076975.089884:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913385856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076975.089887:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385984:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076975.089897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076975.089902:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076975/real 1769076975] req@ffff9350103fb800 x1855008913385984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076975.089909:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913385984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076975.089915:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076975.089933:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076975.089935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913385984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076975.089939:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913386112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076975.089950:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076975.089957:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076975/real 1769076975] req@ffff9350103f9f80 x1855008913386112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076975.089965:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913386112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076975.089971:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076975.089973:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076975.089975:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913386112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076975.089991:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913386240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076975.090002:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076975.090027:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076975/real 1769076975] req@ffff9350103fb480 x1855008913386240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076975.090038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913386240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076975.090045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076975.090047:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076975.090050:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913386240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076975.281205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076975.793114:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076975.793115:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076975.793121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076975.793181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076976.113156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076976.305171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076976.817165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076976.817172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076976.817194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769076976.817252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076977.137141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076977.329172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076977.841139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076977.841139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076977.841139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076977.841179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076978.161144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076978.353141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076978.865141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076978.865141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076978.865155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076978.865176:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076979.185123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076979.377198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076979.889137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076979.889156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076979.889163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076979.889210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076980.209204:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076980.209215:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076980.209220:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076980.209224:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076980.209232:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2685 00000100:00080000:1.0:1769076980.209244:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076980.209266:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076980.209313:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913386368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076980.209363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076980.209368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076980.209371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076980.209376:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2685 00000100:00080000:1.0:1769076980.209379:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076980.209384:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076980.209394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076980.209422:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913386496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076980.209435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076980.209439:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076980.209458:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076980.209477:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913386368:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076980.209483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2685 00000100:00080000:1.0:1769076980.209489:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076980.209494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076980.209500:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076980.209516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913386624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076980.209525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076980.209528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076980.209530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076980.209533:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2685 00000100:00080000:1.0:1769076980.209536:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076980.209539:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076980.209544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769076980.209544:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076980.209557:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913386752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076980.209564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076980.209567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076980.209568:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076980.209572:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2685 00000100:00080000:1.0:1769076980.209575:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076980.209579:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076980.209583:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076980/real 1769076980] req@ffff9350100ca680 x1855008913386368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076980.209584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076980.209597:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913386880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076980.209605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076980.209605:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913386368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076980.209614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076980.209617:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076980.209624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2685 00000100:00080000:1.0:1769076980.209628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076980.209628:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076980.209634:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769076980.209637:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769076980.209639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076980.209643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913386368:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076980.209650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913387008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076980.209657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913386496:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076980.209659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076980.209666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076980.209667:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076980.209671:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2685 00000100:00080000:1.0:1769076980.209673:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076980.209676:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076980.209689:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076980.209698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913387136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076980.209704:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076980.209718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076980/real 1769076980] req@ffff9350100c9f80 x1855008913386496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076980.209729:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913386496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076980.209737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076980.209739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076980.209742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913386496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076980.209762:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913386624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076980.209774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076980.209780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076980/real 1769076980] req@ffff9350100cad80 x1855008913386624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076980.209788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913386624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076980.209794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076980.209796:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076980.209798:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913386624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076980.209801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913386752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076980.209810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076980.209815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076980/real 1769076980] req@ffff9350100c8000 x1855008913386752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076980.209823:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913386752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076980.209828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076980.209830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076980.209832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913386752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076980.209835:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913386880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769076980.209845:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076980.209851:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076980/real 1769076980] req@ffff9350100cb100 x1855008913386880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076980.209859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913386880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076980.209864:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076980.209866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076980.209867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913386880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076980.209871:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076980.209897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076980.209903:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076980/real 1769076980] req@ffff9350100c9180 x1855008913387008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076980.209911:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913387008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076980.209921:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076980.209923:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076980.209926:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076980.209929:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769076980.209938:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769076980.209944:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076980/real 1769076980] req@ffff9350100c9c00 x1855008913387136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076980.209951:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913387136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076980.209957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076980.209959:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769076980.209960:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076980.401167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076980.913159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076980.913159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076980.913169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076980.913192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076981.233154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076981.425156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076981.937156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076981.937156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076981.937190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076981.937270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076982.257131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076982.449147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076982.961131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076982.961138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076982.961160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076982.961164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076983.281239:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076983.473143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076983.985152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076983.985165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076983.985201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076983.985227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076984.305130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076984.497132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076985.009218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076985.009218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076985.009300:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076985.009304:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076985.329141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076985.329151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1769076985.329151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076985.329157:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076985.329167:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2691 00000100:00080000:1.0:1769076985.329177:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076985.329184:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076985.329218:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913387264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076985.329242:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076985.329245:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076985.329246:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076985.329250:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2691 00000100:00080000:1.0:1769076985.329252:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076985.329255:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076985.329263:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076985.329275:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913387392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076985.329280:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913387264:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076985.329287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076985.329294:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076985.329297:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076985.329300:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2691 00000100:00080000:1.0:1769076985.329302:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076985.329305:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076985.329308:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076985.329318:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913387520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076985.329324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076985.329326:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076985.329328:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076985.329330:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2691 00000100:00080000:1.0:1769076985.329334:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076985.329336:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076985.329339:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076985.329346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913387648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076985.329353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076985.329368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076985.329369:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076985.329373:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2691 00000100:00080000:1.0:1769076985.329376:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076985.329379:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076985.329383:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076985.329394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913387776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076985.329402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076985.329405:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076985.329406:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076985.329410:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913387392:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769076985.329412:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2691 00000100:00080000:1.0:1769076985.329418:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076985.329423:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076985.329427:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076985.329438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913387904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076985.329447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076985.329448:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913387520:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769076985.329453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076985.329456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076985.329460:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2691 00000100:00080000:1.0:1769076985.329464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076985.329468:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769076985.329472:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913387648:192.168.204.121@tcp:8:kworker.0 00000008:00100000:1.0:1769076985.329474:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076985.329492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913388032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076985.329493:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769076985.329517:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769076985.329535:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076985.521137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076986.033144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076986.033147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076986.033170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076986.033206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076986.353405:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076986.545155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076987.057214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076987.057231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076987.057249:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076987.057250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076987.377397:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076987.569193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076988.081198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076988.081205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076988.081216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076988.081278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076988.401421:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769076988.401610:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769076988.401683:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076988.401716:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076988.401797:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076988.401825:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076988.401844:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076988.401861:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076988.401880:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769076988.401903:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769076988.401909:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076985/real 1769076988] req@ffff9350100c9c00 x1855008913387392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076988.401977:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913387392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076988.402000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076988.402011:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076988.402020:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913387392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076988.402037:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076985/real 1769076988] req@ffff9350100c9180 x1855008913387520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076988.402126:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913387520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076988.402143:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076988.402155:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076988.402163:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913387520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076988.402181:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076985/real 1769076988] req@ffff9350100cb100 x1855008913387648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076988.402207:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913387648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076988.402225:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076988.402233:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076988.402240:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913387648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076988.402255:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076985/real 1769076988] req@ffff9350100c8000 x1855008913387776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076988.402284:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913387776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076988.402301:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076988.402338:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076988.402346:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076988.402359:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076985/real 1769076988] req@ffff9350100c8a80 x1855008913387264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076988.402387:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913387264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076988.402403:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076988.402411:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076988.402418:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913387264:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769076988.402468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076985/real 1769076988] req@ffff9350100cad80 x1855008913387904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076988.402490:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913387904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076988.402504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076988.402510:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076988.402517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913387904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076988.402530:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076985/real 1769076988] req@ffff9350100c9f80 x1855008913388032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076988.402584:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913388032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076988.402599:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076988.402606:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076988.402614:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076988.593162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076989.105228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076989.105281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076989.105340:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076989.105392:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076989.425340:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076989.617188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076990.129253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076990.129254:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076990.129346:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076990.129355:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076990.449252:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076990.449267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076990.449311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076990.449319:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076990.449335:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2696 00000100:00080000:1.0:1769076990.449353:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076990.449375:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076990.449438:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913388160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076990.449490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076990.449496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076990.449499:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076990.449505:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2696 00000100:00080000:1.0:1769076990.449509:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076990.449514:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076990.449529:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076990.449546:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913388288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076990.449571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076990.449577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076990.449579:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076990.449584:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2696 00000100:00080000:1.0:1769076990.449589:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076990.449593:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076990.449603:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769076990.449610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913388160:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769076990.449622:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913388416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076990.449638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076990.449643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076990.449645:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076990.449650:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2696 00000100:00080000:1.0:1769076990.449654:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076990.449658:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076990.449664:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076990.449680:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913388544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076990.449693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076990.449700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076990.449704:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076990.449713:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2696 00000100:00080000:1.0:1769076990.449722:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076990.449730:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076990.449740:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076990.449752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076990.449775:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913388672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076990.449802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076990.449860:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076990/real 1769076990] req@ffff9350100ca680 x1855008913388160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076990.449863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076990.449871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076990.449886:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2696 00000100:00080000:1.0:1769076990.449897:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076990.449905:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076990.449914:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076990.449938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913388800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076990.449947:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913388160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076990.449967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076990.449983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076990.449988:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076990.449992:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076990.449999:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2696 00000100:00080000:1.0:1769076990.450008:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076990.450009:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769076990.450022:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769076990.450028:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913388160:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769076990.450035:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076990.450049:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913388288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076990.450130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913388928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076990.450166:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076990.450200:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076990/real 1769076990] req@ffff9350100c8e00 x1855008913388288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076990.450226:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913388288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076990.450252:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076990.450259:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076990.450265:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913388288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076990.450337:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913388416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076990.450375:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076990.450396:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076990/real 1769076990] req@ffff9350100c9880 x1855008913388416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076990.450416:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913388416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076990.450429:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076990.450435:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076990.450440:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913388416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076990.450450:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913388544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076990.450481:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076990.450500:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076990/real 1769076990] req@ffff9350100c9500 x1855008913388544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076990.450520:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913388544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076990.450536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076990.450541:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076990.450548:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913388544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076990.450559:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388672:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076990.450594:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076990.450615:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076990/real 1769076990] req@ffff9350100caa00 x1855008913388672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076990.450637:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913388672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076990.450652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076990.450659:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076990.450665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076990.450677:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076990.450713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076990.450739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076990/real 1769076990] req@ffff9350100c8380 x1855008913388800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076990.450762:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913388800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076990.450776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076990.450782:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076990.450788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076990.450800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076990.450834:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076990.450854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076990/real 1769076990] req@ffff9350100ca300 x1855008913388928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076990.450874:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913388928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076990.450888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076990.450893:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076990.450899:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913388928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076990.641113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076991.153213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076991.153221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076991.153286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076991.153289:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076991.473331:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076991.665163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076992.177266:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076992.177278:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076992.177285:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076992.177347:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076992.497159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076992.689123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076993.201132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076993.201135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076993.201172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076993.201184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076993.521124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076993.713166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076994.225187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076994.225226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076994.225255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076994.225264:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076994.545162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076994.737163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076995.249182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076995.249221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076995.249297:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076995.249305:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769076995.569134:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076995.569138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769076995.569147:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076995.569152:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076995.569160:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2701 00000100:00080000:1.0:1769076995.569171:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076995.569177:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769076995.569274:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913389056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076995.569304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076995.569308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076995.569310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076995.569315:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2701 00000100:00080000:1.0:1769076995.569319:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076995.569324:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076995.569335:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076995.569351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913389184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076995.569360:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913389056:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076995.569366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076995.569375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076995.569377:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076995.569381:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2701 00000100:00080000:1.0:1769076995.569384:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076995.569387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076995.569392:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769076995.569406:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913389312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076995.569416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076995.569419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076995.569420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076995.569425:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2701 00000100:00080000:1.0:1769076995.569441:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076995.569446:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076995.569451:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769076995.569461:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769076995.569466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913389440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076995.569479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076995.569484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076995.569485:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076995.569488:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2701 00000100:00080000:1.0:1769076995.569501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076995.569523:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076995.569528:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076995.569551:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076995/real 1769076995] req@ffff9350100cb800 x1855008913389056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076995.569561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913389568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076995.569572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076995.569573:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913389056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076995.569583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076995.569589:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076995.569593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076995.569598:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2701 00000100:00080000:0.0:1769076995.569601:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076995.569608:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913389056:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769076995.569609:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076995.569620:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769076995.569627:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076995.569635:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913389184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769076995.569646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913389696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076995.569659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076995.569664:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769076995.569669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076995.569674:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076995.569681:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076995/real 1769076995] req@ffff9350100ca300 x1855008913389184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076995.569682:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2701 00000100:00080000:1.0:1769076995.569703:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076995.569711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913389184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076995.569714:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769076995.569728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769076995.569729:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076995.569733:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076995.569738:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913389184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076995.569744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913389312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769076995.569757:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913389824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076995.569771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076995.569783:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076995/real 1769076995] req@ffff9350100c8380 x1855008913389312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076995.569794:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913389312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076995.569800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076995.569803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076995.569805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913389312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076995.569809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913389440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076995.569840:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076995.569849:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076995/real 1769076995] req@ffff9350100caa00 x1855008913389440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076995.569861:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913389440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076995.569868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076995.569881:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076995.569887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913389440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076995.569896:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913389568:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769076995.569924:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076995.569960:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076995/real 1769076995] req@ffff9350100c9500 x1855008913389568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076995.569985:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913389568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076995.570000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076995.570006:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076995.570012:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913389568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769076995.570043:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913389696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076995.570129:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076995.570160:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076995/real 1769076995] req@ffff9350100c9880 x1855008913389696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076995.570196:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913389696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076995.570207:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076995.570211:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076995.570215:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913389696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769076995.570222:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913389824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769076995.570247:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769076995.570258:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076995/real 1769076995] req@ffff9350100c8e00 x1855008913389824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076995.570276:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913389824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076995.570282:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076995.570285:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769076995.570287:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913389824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769076995.761210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076996.273231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076996.273236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076996.273279:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076996.273290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076996.593161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076996.785245:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076997.297231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076997.297247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076997.297283:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076997.297334:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076997.617230:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076997.809192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769076998.321241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076998.321264:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076998.321264:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769076998.321303:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076998.641246:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076998.833294:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769076999.345204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769076999.345236:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076999.345260:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076999.345270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076999.665293:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769076999.857168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077000.369243:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077000.369323:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077000.369327:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077000.369328:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769077000.689230:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077000.689243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077000.689249:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077000.689260:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2706 00000100:00080000:3.0:1769077000.689290:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077000.689299:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077000.689351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913389952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077000.689384:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077000.689388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077000.689389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077000.689394:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2706 00000100:00080000:3.0:1769077000.689397:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077000.689400:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077000.689412:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077000.689424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913390080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077000.689433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077000.689437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077000.689438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077000.689442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2706 00000100:00080000:3.0:1769077000.689445:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077000.689447:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077000.689454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077000.689463:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913390208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077000.689473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077000.689477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077000.689479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077000.689483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2706 00000100:00080000:3.0:1769077000.689485:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077000.689488:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077000.689492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077000.689528:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913389952:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077000.689531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913390336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077000.689554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077000.689557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077000.689558:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077000.689564:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2706 00000100:00080000:3.0:1769077000.689568:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077000.689572:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077000.689578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077000.689592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913390464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077000.689601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077000.689603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077000.689605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077000.689609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2706 00000100:00080000:3.0:1769077000.689614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077000.689619:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077000.689622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077000.689636:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077000.689637:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913390592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077000.689671:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077000.689674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077000.689677:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077000.689683:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2706 00000100:00080000:3.0:1769077000.689687:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077000.689691:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077000.689698:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077000.689709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913390720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077000.689721:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077000/real 1769077000] req@ffff935002a7e300 x1855008913389952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077000.689770:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077000.689779:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913389952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077000.689792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077000.689799:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077000.689805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913389952:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077000.689814:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390080:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077000.689841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077000.689854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077000/real 1769077000] req@ffff935002a7ca80 x1855008913390080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077000.689865:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913390080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077000.689873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077000.689876:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077000.689878:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077000.689904:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077000.689919:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077000.689936:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077000/real 1769077000] req@ffff935002a7c380 x1855008913390208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077000.689963:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913390208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077000.689978:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077000.689982:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077000.689986:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077000.689992:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077000.690028:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077000.690044:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077000/real 1769077000] req@ffff935002a7f100 x1855008913390336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077000.690161:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913390336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077000.690174:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077000.690179:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077000.690183:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077000.690191:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913390464:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077000.690220:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077000.690235:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077000/real 1769077000] req@ffff935002a7c700 x1855008913390464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077000.690262:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913390464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077000.690269:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077000.690271:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077000.690275:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913390464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077000.690280:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913390592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077000.690297:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077000.690304:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077000/real 1769077000] req@ffff935002a7f800 x1855008913390592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077000.690334:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913390592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077000.690341:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077000.690344:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077000.690347:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913390592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077000.690352:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913390720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077000.690366:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077000.690374:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077000/real 1769077000] req@ffff935002a7c000 x1855008913390720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077000.690383:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913390720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077000.690389:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077000.690391:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077000.690394:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913390720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077000.881276:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077001.393275:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077001.393275:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077001.393276:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077001.393319:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077001.713215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077001.905125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077002.417207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077002.417215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077002.417243:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077002.417266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077002.737177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077002.929268:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077003.441268:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077003.441269:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077003.441269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077003.441319:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077003.761164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077003.953132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077004.465175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077004.465179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077004.465179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077004.465243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077004.785226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077004.977210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077005.489250:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077005.489250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077005.489283:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077005.489342:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077005.809222:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077005.809306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077005.809322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077005.809327:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077005.809338:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2711 00000100:00080000:1.0:1769077005.809351:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077005.809360:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077005.809414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913390848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077005.809457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077005.809474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077005.809476:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077005.809483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2711 00000100:00080000:1.0:1769077005.809489:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077005.809496:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077005.809510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077005.809547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913390976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077005.809585:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913390848:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077005.809586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077005.809615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077005.809625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077005.809634:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2711 00000100:00080000:1.0:1769077005.809645:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077005.809654:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077005.809665:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077005.809691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913391104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077005.809722:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077005.809728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077005.809732:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077005.809746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077005.809757:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2711 00000100:00080000:1.0:1769077005.809769:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077005.809788:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077005.809799:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077005.809823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913391232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077005.809841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077005.809849:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077005/real 1769077005] req@ffff9350100ca680 x1855008913390848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077005.809853:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077005.809868:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077005.809888:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2711 00000100:00080000:1.0:1769077005.809900:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077005.809909:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077005.809916:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913390848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077005.809959:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077005.809963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077005.809978:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077005.809993:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913390848:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077005.810004:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913391360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077005.810168:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390976:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077005.810168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077005.810193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077005.810199:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077005.810211:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2711 00000100:00080000:1.0:1769077005.810222:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077005.810230:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077005.810241:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077005.810255:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077005.810256:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077005/real 1769077005] req@ffff9350100c9f80 x1855008913390976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077005.810299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913391488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077005.810309:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913390976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077005.810350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077005.810357:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077005.810360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077005.810374:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077005.810381:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077005.810393:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2711 00000100:00100000:0.0:1769077005.810396:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913390976:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077005.810405:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077005.810421:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913391104:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769077005.810421:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077005.810441:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077005.810470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913391616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077005.810507:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077005.810536:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077005/real 1769077005] req@ffff9350100cad80 x1855008913391104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077005.810571:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913391104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077005.810582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077005.810587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077005.810591:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913391104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077005.810600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913391232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077005.810621:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077005.810632:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077005/real 1769077005] req@ffff9350100c8a80 x1855008913391232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077005.810668:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913391232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077005.810678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077005.810681:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077005.810685:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913391232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077005.810709:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913391360:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077005.810753:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077005.810771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077005/real 1769077005] req@ffff9350100c8000 x1855008913391360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077005.810804:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913391360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077005.810819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077005.810824:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077005.810830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913391360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077005.810839:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913391488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077005.810883:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077005.810896:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077005/real 1769077005] req@ffff9350100cb100 x1855008913391488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077005.810928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913391488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077005.810937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077005.810941:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077005.810945:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913391488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077005.810953:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913391616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077005.810972:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077005.810987:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077005/real 1769077005] req@ffff9350100c9180 x1855008913391616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077005.811043:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913391616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077005.811101:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077005.811108:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077005.811112:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913391616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077006.001144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077006.513200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077006.513202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077006.513202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077006.513240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077006.833190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077007.025129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077007.537147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077007.537147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077007.537170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077007.537201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077007.857135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077008.049172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077008.561145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077008.561145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077008.561148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077008.561281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077008.881172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077009.073144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077009.585128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077009.585136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077009.585136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077009.585148:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077009.905249:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077010.097160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077010.609123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077010.609128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077010.609131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077010.609168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077010.929184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077010.929186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.929196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077010.929200:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.929209:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2716 00000100:00080000:3.0:1769077010.929221:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.929228:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077010.929269:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913391744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.929300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.929303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077010.929305:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.929308:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2716 00000100:00080000:3.0:1769077010.929312:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.929316:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077010.929324:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077010.929347:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913391744:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077010.929349:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913391872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.929361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.929363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077010.929365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.929368:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2716 00000100:00080000:3.0:1769077010.929371:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.929373:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077010.929376:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077010.929385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913392000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.929392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.929394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077010.929395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.929397:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2716 00000100:00080000:3.0:1769077010.929399:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.929402:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077010.929404:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077010.929410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913392128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.929416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.929419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077010.929420:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077010.929424:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077010.929432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2716 00000100:00080000:3.0:1769077010.929436:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.929439:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077010.929444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077010.929455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913392256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.929462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077010.929465:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff935002a7fb80 x1855008913391744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077010.929466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077010.929469:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.929476:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2716 00000100:00080000:3.0:1769077010.929482:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.929487:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077010.929491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077010.929495:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913391744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077010.929501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913392384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077010.929509:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077010.929511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.929515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077010.929516:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077010.929517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077010.929525:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913391744:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077010.929525:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2716 00000100:00080000:3.0:1769077010.929530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.929535:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077010.929539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077010.929545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913392512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077010.929554:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913391872:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077010.929573:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077010.929583:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff935002a7dc00 x1855008913391872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.929591:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913391872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.929598:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.929600:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077010.929602:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913391872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077010.929606:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077010.929617:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077010.929623:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff935002a7d180 x1855008913392000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.929630:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913392000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.929635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.929637:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077010.929639:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077010.929642:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077010.929651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077010.929656:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff935002a7df80 x1855008913392128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.929663:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913392128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.929669:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.929671:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077010.929674:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077010.929677:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913392256:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077010.929686:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077010.929691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff935002a7c000 x1855008913392256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.929697:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913392256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.929712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.929715:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077010.929717:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913392256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077010.929720:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913392384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077010.929729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077010.929734:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff935002a7f800 x1855008913392384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.929741:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913392384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.929745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.929747:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077010.929750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913392384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077010.929763:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913392512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077010.929773:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077010.929779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff935002a7c700 x1855008913392512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.929786:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913392512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.929792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.929793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077010.929795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913392512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077011.121152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077011.633154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077011.633154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077011.633154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077011.633192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077011.953136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077012.145128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077012.657172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077012.657177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077012.657182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077012.657233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077012.977216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077013.169134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077013.681187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077013.681188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077013.681188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077013.681216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077014.001160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077014.193117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077014.705133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077014.705133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077014.705133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077014.705167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077015.025116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077015.217152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077015.729231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077015.729240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077015.729275:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077015.729332:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077016.049156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077016.049167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077016.049172:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077016.049176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077016.049182:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2721 00000100:00080000:3.0:1769077016.049195:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077016.049202:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077016.049240:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913392640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077016.049302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077016.049306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077016.049309:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077016.049315:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2721 00000100:00080000:3.0:1769077016.049319:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077016.049326:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077016.049339:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077016.049368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913392768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077016.049377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077016.049380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077016.049382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077016.049385:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2721 00000100:00080000:3.0:1769077016.049388:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077016.049394:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077016.049395:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913392640:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077016.049398:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077016.049410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913392896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077016.049418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077016.049421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077016.049423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077016.049426:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2721 00000100:00080000:3.0:1769077016.049428:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077016.049431:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077016.049435:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077016.049443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913393024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077016.049451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077016.049454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077016.049455:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077016.049459:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2721 00000100:00080000:3.0:1769077016.049465:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077016.049468:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077016.049471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077016.049481:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077016.049485:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913393152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077016.049494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077016.049498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077016.049500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077016.049503:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2721 00000100:00080000:3.0:1769077016.049513:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077016.049516:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077016.049520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077016.049532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913393280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077016.049557:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077016/real 1769077016] req@ffff935002a7f100 x1855008913392640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077016.049563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077016.049577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077016.049582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077016.049593:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2721 00000100:00080000:3.0:1769077016.049598:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077016.049604:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077016.049610:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913392640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077016.049614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077016.049634:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077016.049635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913393408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077016.049645:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077016.049668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913392640:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077016.049678:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392768:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077016.049710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077016.049725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077016/real 1769077016] req@ffff935002a7c380 x1855008913392768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077016.049739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913392768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077016.049761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077016.049765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077016.049768:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077016.049814:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077016.049830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077016.049837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077016/real 1769077016] req@ffff935002a7ca80 x1855008913392896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077016.049868:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913392896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077016.049876:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077016.049889:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077016.049892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913392896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077016.049897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077016.049913:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077016.049922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077016/real 1769077016] req@ffff935002a7e300 x1855008913393024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077016.049931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913393024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077016.049938:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077016.049941:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077016.049944:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077016.049948:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913393152:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077016.049959:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077016.049966:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077016/real 1769077016] req@ffff935002a7c700 x1855008913393152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077016.049981:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913393152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077016.049991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077016.049995:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077016.049999:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913393152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077016.050007:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913393280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077016.050034:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077016.050091:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077016/real 1769077016] req@ffff935002a7f800 x1855008913393280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077016.050104:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913393280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077016.050112:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077016.050116:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077016.050118:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913393280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077016.050123:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913393408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077016.050142:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077016.050154:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077016/real 1769077016] req@ffff935002a7c000 x1855008913393408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077016.050169:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913393408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077016.050179:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077016.050183:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077016.050188:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913393408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077016.241181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077016.753121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077016.753125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077016.753125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077016.753157:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077017.073141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077017.265119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077017.777148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077017.777164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077017.777185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077017.777259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077018.097264:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077018.289200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077018.801230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077018.801230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077018.801230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077018.801254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077019.121183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077019.313176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077019.825165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077019.825167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077019.825175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077019.825199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077020.145197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077020.337219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077020.849193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077020.849197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077020.849197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077020.849260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077021.169145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077021.169153:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077021.169163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077021.169168:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077021.169179:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2726 00000100:00080000:3.0:1769077021.169192:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077021.169199:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077021.169243:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913393536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077021.169269:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077021.169273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077021.169274:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077021.169278:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2726 00000100:00080000:3.0:1769077021.169280:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077021.169284:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077021.169294:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077021.169304:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913393664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077021.169312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077021.169315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077021.169316:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077021.169321:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2726 00000100:00100000:0.0:1769077021.169323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913393536:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077021.169323:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077021.169329:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077021.169335:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077021.169342:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913393792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077021.169348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077021.169351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077021.169352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077021.169355:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2726 00000100:00080000:3.0:1769077021.169357:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077021.169359:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077021.169362:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077021.169382:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913393920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077021.169387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077021.169390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077021.169392:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077021.169395:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2726 00000100:00080000:3.0:1769077021.169398:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077021.169401:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077021.169403:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077021.169410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913394048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077021.169416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077021.169418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077021.169420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077021.169422:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2726 00000100:00080000:3.0:1769077021.169425:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077021.169427:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077021.169431:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077021.169432:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077021.169441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913394176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077021.169448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077021.169450:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077021.169452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077021.169455:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2726 00000100:00080000:3.0:1769077021.169458:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077021.169460:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077021.169464:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077021.169470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913394304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077021.169492:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077021/real 1769077021] req@ffff935002a7df80 x1855008913393536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077021.169521:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913393536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077021.169531:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077021.169534:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077021.169538:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913393536:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077021.169561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393664:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077021.169578:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077021.169587:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077021/real 1769077021] req@ffff935002a7d180 x1855008913393664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077021.169596:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913393664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077021.169614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077021.169617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077021.169619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077021.169624:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077021.169635:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077021.169641:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077021/real 1769077021] req@ffff935002a7dc00 x1855008913393792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077021.169649:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913393792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077021.169655:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077021.169658:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077021.169662:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077021.169669:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077021.169688:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077021.169700:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077021/real 1769077021] req@ffff935002a7fb80 x1855008913393920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077021.169714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913393920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077021.169722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077021.169724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077021.169727:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913393920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077021.169731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394048:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077021.169747:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077021.169770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077021/real 1769077021] req@ffff935002a7c000 x1855008913394048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077021.169779:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913394048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077021.169785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077021.169787:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077021.169790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077021.169794:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077021.169809:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077021.169816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077021/real 1769077021] req@ffff935002a7f800 x1855008913394176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077021.169825:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913394176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077021.169840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077021.169842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077021.169844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077021.169847:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077021.169856:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077021.169861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077021/real 1769077021] req@ffff935002a7c700 x1855008913394304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077021.169869:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913394304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077021.169875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077021.169876:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077021.169878:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077021.361146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077021.873240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077021.873240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077021.873246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077021.873246:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077022.193158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077022.385138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077022.897165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077022.897234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077022.897236:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077022.897240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077023.217166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077023.409118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077023.921189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077023.921191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077023.921215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077023.921277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077024.241188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077024.433121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077024.945167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077024.945167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077024.945167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077024.945204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077025.265136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077025.457110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077025.969155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077025.969155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077025.969155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077025.969194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077026.289147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077026.289151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077026.289159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077026.289164:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077026.289171:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2731 00000100:00080000:3.0:1769077026.289182:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077026.289188:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077026.289220:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913394432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077026.289242:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077026.289245:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077026.289246:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077026.289251:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2731 00000100:00080000:3.0:1769077026.289255:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077026.289260:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077026.289270:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077026.289285:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913394560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077026.289288:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913394432:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077026.289307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077026.289311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077026.289313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077026.289317:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2731 00000100:00080000:3.0:1769077026.289320:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077026.289324:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077026.289328:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077026.289339:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913394688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077026.289346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077026.289348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077026.289349:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077026.289352:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2731 00000100:00080000:3.0:1769077026.289355:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077026.289356:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077026.289360:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077026.289365:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077026.289373:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913394816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077026.289380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077026.289383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077026.289384:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077026.289388:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2731 00000100:00080000:3.0:1769077026.289391:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077026.289394:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077026.289397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077026.289400:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077026/real 1769077026] req@ffff935002a7e300 x1855008913394432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077026.289408:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913394944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077026.289437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077026.289440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077026.289442:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077026.289446:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913394432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077026.289447:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2731 00000100:00080000:3.0:1769077026.289452:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077026.289458:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077026.289464:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077026.289465:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077026.289473:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077026.289477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913395072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077026.289479:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913394432:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077026.289492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077026.289496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077026.289498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077026.289502:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2731 00000100:00080000:3.0:1769077026.289506:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077026.289507:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913394560:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077026.289515:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077026.289522:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077026.289532:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077026.289541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913395200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077026.289553:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077026/real 1769077026] req@ffff935002a7ca80 x1855008913394560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077026.289566:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913394560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077026.289576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077026.289580:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077026.289584:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913394560:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077026.289590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913394688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077026.289612:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077026.289624:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077026/real 1769077026] req@ffff935002a7c380 x1855008913394688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077026.289637:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913394688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077026.289647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077026.289650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077026.289652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913394688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077026.289656:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913394816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077026.289695:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077026.289704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077026/real 1769077026] req@ffff935002a7f100 x1855008913394816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077026.289712:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913394816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077026.289724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077026.289726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077026.289729:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913394816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077026.289732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394944:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077026.289744:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077026.289750:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077026/real 1769077026] req@ffff935002a7c700 x1855008913394944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077026.289758:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913394944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077026.289763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077026.289765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077026.289769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913394944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077026.289774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077026.289792:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077026.289803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077026/real 1769077026] req@ffff935002a7f800 x1855008913395072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077026.289826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913395072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077026.289834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077026.289838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077026.289842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077026.289859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077026.289874:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077026.289881:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077026/real 1769077026] req@ffff935002a7c000 x1855008913395200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077026.289889:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913395200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077026.289894:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077026.289896:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077026.289898:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077026.481185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077026.993114:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077026.993128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077026.993180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077026.993206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077027.313141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077027.505103:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077028.017184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077028.017184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077028.017273:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077028.017294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077028.337152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077028.529177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077029.041188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077029.041188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077029.041247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077029.041259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077029.361145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077029.553197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077030.065141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077030.065147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077030.065165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077030.065172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077030.385178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077030.577128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077031.089183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077031.089211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077031.089232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077031.089233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:1.0:1769077031.409191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077031.409204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077031.409210:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077031.409212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077031.409234:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2737 00000100:00080000:1.0:1769077031.409250:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077031.409258:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077031.409309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913395328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077031.409340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077031.409344:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077031.409346:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077031.409356:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2737 00000100:00080000:1.0:1769077031.409359:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077031.409363:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077031.409375:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077031.409402:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913395456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077031.409413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077031.409420:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913395328:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077031.409420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077031.409425:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077031.409436:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2737 00000100:00080000:1.0:1769077031.409439:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077031.409442:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077031.409447:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077031.409459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913395584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077031.409468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077031.409473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077031.409475:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077031.409479:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2737 00000100:00080000:1.0:1769077031.409494:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077031.409498:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077031.409502:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077031.409513:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077031.409518:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913395712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077031.409529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077031.409533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077031.409534:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077031.409539:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2737 00000100:00080000:1.0:1769077031.409543:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077031.409547:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077031.409550:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077031.409559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913395840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077031.409567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077031.409571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077031.409573:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077031.409577:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2737 00000100:00080000:1.0:1769077031.409603:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077031.409608:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077031.409633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077031/real 1769077031] req@ffff9350100c9c00 x1855008913395328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077031.409648:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077031.409672:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913395968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077031.409675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913395328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077031.409707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077031.409712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077031.409713:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077031.409716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077031.409724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077031.409726:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2737 00000100:00080000:1.0:1769077031.409735:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077031.409737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913395328:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077031.409744:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077031.409752:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077031.409765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913396096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077031.409778:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913395456:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077031.409813:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077031.409840:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077031/real 1769077031] req@ffff9350100c9180 x1855008913395456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077031.409854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913395456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077031.409864:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077031.409868:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077031.409872:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913395456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077031.409879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913395584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077031.409896:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077031.409907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077031/real 1769077031] req@ffff9350100cb100 x1855008913395584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077031.409919:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913395584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077031.409928:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077031.409931:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077031.409934:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913395584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077031.409939:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913395712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077031.409955:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077031.409976:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077031/real 1769077031] req@ffff9350100c8000 x1855008913395712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077031.409988:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913395712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077031.409997:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077031.409999:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077031.410002:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913395712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077031.410008:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395840:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077031.410041:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077031.410049:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077031/real 1769077031] req@ffff9350100c8a80 x1855008913395840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077031.410060:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913395840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077031.410068:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077031.410071:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077031.410074:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395840:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077031.410120:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077031.410139:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077031.410149:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077031/real 1769077031] req@ffff9350100cad80 x1855008913395968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077031.410159:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913395968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077031.410167:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077031.410170:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077031.410173:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913395968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077031.410192:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077031.410206:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077031.410216:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077031/real 1769077031] req@ffff9350100c9f80 x1855008913396096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077031.410237:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913396096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077031.410243:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077031.410246:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077031.410250:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077031.601177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077032.113145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077032.113157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077032.113167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077032.113199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077032.433226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077032.625171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077033.137209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077033.137209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077033.137244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077033.137313:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077033.457177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077033.649184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077034.161130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077034.161137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077034.161148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077034.161169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077034.481183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077034.673166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077035.185127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077035.185127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077035.185141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077035.185187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077035.505167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077035.697113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077036.209149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077036.209182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077036.209205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077036.209243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077036.529157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077036.529171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077036.529177:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077036.529177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077036.529188:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2742 00000100:00080000:1.0:1769077036.529200:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077036.529208:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077036.529242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913396224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077036.529265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077036.529268:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077036.529269:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077036.529273:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2742 00000100:00080000:1.0:1769077036.529275:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077036.529278:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077036.529285:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077036.529292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913396352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077036.529299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077036.529301:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077036.529302:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077036.529305:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2742 00000100:00080000:1.0:1769077036.529306:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077036.529308:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077036.529311:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077036.529317:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913396480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077036.529323:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077036.529325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077036.529328:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077036.529332:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2742 00000100:00080000:1.0:1769077036.529336:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077036.529340:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913396224:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077036.529347:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077036.529366:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913396608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077036.529377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077036.529381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077036.529384:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077036.529388:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2742 00000100:00080000:1.0:1769077036.529402:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077036.529406:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077036.529410:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077036.529414:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077036.529423:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913396736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077036.529431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077036.529433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077036.529435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077036.529438:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2742 00000100:00080000:1.0:1769077036.529440:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077036.529442:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077036.529446:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077036.529453:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077036/real 1769077036] req@ffff9350100ca680 x1855008913396224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077036.529457:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913396864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077036.529477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913396224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077036.529484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077036.529488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077036.529490:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077036.529491:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077036.529509:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2742 00000100:00080000:0.0:1769077036.529513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077036.529515:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077036.529519:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913396224:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077036.529523:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529529:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913396352:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769077036.529530:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077036.529547:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913396992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077036.529556:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529568:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077036/real 1769077036] req@ffff9350100c8e00 x1855008913396352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077036.529577:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913396352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077036.529583:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077036.529586:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077036.529588:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913396352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077036.529593:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913396480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077036.529604:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529609:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077036/real 1769077036] req@ffff9350100c9880 x1855008913396480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077036.529617:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913396480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077036.529622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077036.529624:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077036.529626:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913396480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077036.529645:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913396608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077036.529666:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077036/real 1769077036] req@ffff9350100c9500 x1855008913396608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077036.529680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913396608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077036.529685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077036.529686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077036.529688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913396608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077036.529691:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396736:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077036.529700:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529707:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077036/real 1769077036] req@ffff9350100caa00 x1855008913396736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077036.529714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913396736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077036.529719:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077036.529721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077036.529723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077036.529726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077036.529735:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077036/real 1769077036] req@ffff9350100c8380 x1855008913396864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077036.529748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913396864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077036.529753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077036.529755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077036.529756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077036.529760:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077036.529774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077036.529780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077036/real 1769077036] req@ffff9350100ca300 x1855008913396992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077036.529787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913396992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077036.529793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077036.529795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077036.529797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913396992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077036.721122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077037.233131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077037.233131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077037.233133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077037.233232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077037.553213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077037.745189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077038.257149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077038.257199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077038.257252:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077038.257384:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077038.577161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077038.769215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077039.281208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077039.281259:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077039.281260:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077039.281334:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077039.601223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077039.793225:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077040.305214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077040.305261:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077040.305299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077040.305306:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077040.625203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077040.817156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077041.329185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077041.329197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077041.329198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077041.329233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077041.649246:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077041.649256:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077041.649268:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077041.649275:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077041.649304:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2747 00000100:00080000:3.0:1769077041.649319:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077041.649328:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077041.649379:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913397120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077041.649441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077041.649446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077041.649449:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077041.649473:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2747 00000100:00080000:3.0:1769077041.649479:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077041.649484:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077041.649497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077041.649516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913397248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077041.649540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077041.649545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077041.649548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077041.649554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2747 00000100:00080000:3.0:1769077041.649559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077041.649563:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077041.649570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077041.649578:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913397120:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077041.649590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913397376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077041.649606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077041.649621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077041.649624:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077041.649628:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2747 00000100:00080000:3.0:1769077041.649633:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077041.649637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077041.649643:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077041.649658:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913397504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077041.649670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077041.649692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077041.649694:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077041.649698:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2747 00000100:00080000:3.0:1769077041.649704:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077041.649712:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077041.649714:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077041.649724:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077041.649744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913397632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077041.649756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077041.649760:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077041.649783:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077041/real 1769077041] req@ffff935002a7fb80 x1855008913397120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077041.649788:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077041.649820:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2747 00000100:00080000:3.0:1769077041.649835:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077041.649840:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913397120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077041.649844:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077041.649857:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077041.649858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077041.649865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077041.649869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913397120:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077041.649879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913397248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077041.649880:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913397760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077041.649892:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077041.649896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077041.649898:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077041.649902:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2747 00000800:00000100:0.0:1769077041.649907:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077041.649908:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077041.649915:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077041.649921:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077041.649923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077041/real 1769077041] req@ffff935002a7dc00 x1855008913397248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077041.649933:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913397888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077041.649937:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913397248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077041.649968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077041.649974:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077041.649979:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913397248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077041.650003:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913397376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077041.650016:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077041.650023:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077041/real 1769077041] req@ffff935002a7d180 x1855008913397376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077041.650031:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913397376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077041.650046:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077041.650048:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077041.650050:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913397376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077041.650054:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913397504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077041.650063:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077041.650069:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077041/real 1769077041] req@ffff935002a7df80 x1855008913397504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077041.650101:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913397504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077041.650108:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077041.650110:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077041.650113:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913397504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077041.650118:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913397632:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077041.650152:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077041.650168:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077041/real 1769077041] req@ffff935002a7c000 x1855008913397632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077041.650183:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913397632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077041.650194:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077041.650197:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077041.650202:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913397632:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077041.650208:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913397760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077041.650227:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077041.650237:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077041/real 1769077041] req@ffff935002a7f800 x1855008913397760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077041.650252:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913397760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077041.650282:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077041.650285:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077041.650288:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913397760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077041.650295:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913397888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077041.650313:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077041.650324:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077041/real 1769077041] req@ffff935002a7c700 x1855008913397888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077041.650337:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913397888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077041.650357:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077041.650359:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077041.650363:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913397888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077041.841177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077042.353196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077042.353199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077042.353231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077042.353351:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077042.673158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077042.865119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077043.377145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077043.377148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077043.377148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077043.377180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077043.697138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077043.889129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077044.401174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077044.401174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077044.401271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077044.401297:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077044.721182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077044.913131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077045.425171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077045.425172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077045.425172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077045.425251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077045.745132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077045.937238:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077046.449199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077046.449199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077046.449370:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077046.449381:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077046.769145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077046.769149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077046.769159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077046.769166:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077046.769180:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2752 00000100:00080000:1.0:1769077046.769192:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077046.769201:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077046.769249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913398016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077046.769280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077046.769285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077046.769287:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077046.769292:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2752 00000100:00080000:1.0:1769077046.769296:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077046.769301:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077046.769314:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077046.769351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913398144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077046.769355:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913398016:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077046.769373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077046.769377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077046.769379:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077046.769386:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2752 00000100:00080000:1.0:1769077046.769390:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077046.769395:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077046.769403:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077046.769416:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913398272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077046.769428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077046.769432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077046.769434:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077046.769438:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2752 00000100:00080000:1.0:1769077046.769443:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077046.769447:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077046.769453:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769077046.769454:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077046.769470:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913398400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077046.769481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077046.769486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077046.769488:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077046.769494:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2752 00000100:00080000:1.0:1769077046.769498:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077046.769503:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077046.769509:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077046/real 1769077046] req@ffff9350100cb800 x1855008913398016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077046.769510:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077046.769530:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913398528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077046.769537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913398016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077046.769554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077046.769558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077046.769560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077046.769565:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077046.769566:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077046.769576:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2752 00000100:00100000:0.0:1769077046.769577:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913398016:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077046.769584:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077046.769592:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077046.769599:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077046.769610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913398144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077046.769614:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913398656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077046.769629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077046.769633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077046.769636:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077046.769643:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077046.769644:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2752 00000100:00080000:1.0:1769077046.769652:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077046.769658:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077046.769665:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077046.769666:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077046/real 1769077046] req@ffff9350100ca300 x1855008913398144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077046.769683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913398784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077046.769686:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913398144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077046.769713:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077046.769719:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077046.769723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913398144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077046.769731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913398272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077046.769753:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077046.769765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077046/real 1769077046] req@ffff9350100c8380 x1855008913398272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077046.769777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913398272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077046.769785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077046.769788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077046.769791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913398272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077046.769797:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913398400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077046.769816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077046.769826:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077046/real 1769077046] req@ffff9350100caa00 x1855008913398400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077046.769838:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913398400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077046.769847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077046.769850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077046.769853:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913398400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077046.769859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913398528:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077046.769877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077046.769896:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077046/real 1769077046] req@ffff9350100c9500 x1855008913398528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077046.769909:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913398528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077046.769918:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077046.769921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077046.769924:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913398528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077046.769942:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913398656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077046.769961:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077046.769971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077046/real 1769077046] req@ffff9350100c9880 x1855008913398656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077046.769984:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913398656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077046.769992:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077046.769995:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077046.770000:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913398656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077046.770006:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913398784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077046.770024:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077046.770036:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077046/real 1769077046] req@ffff9350100c8e00 x1855008913398784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077046.770076:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913398784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077046.770086:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077046.770089:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077046.770093:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913398784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077046.961190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077047.473148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077047.473148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077047.473159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077047.473183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077047.793171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077047.985152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077048.497169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077048.497175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077048.497187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077048.497241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077048.817114:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077049.009172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077049.521161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077049.521167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077049.521169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077049.521213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077049.841147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077050.033122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077050.545154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077050.545154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077050.545167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077050.545196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077050.865107:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077051.057201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077051.569163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077051.569194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077051.569204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077051.569291:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077051.889268:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077051.889297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.889306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077051.889321:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.889334:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2757 00000100:00080000:0.0:1769077051.889349:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.889359:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077051.889422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913398912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077051.889470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.889474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077051.889476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.889480:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2757 00000100:00080000:0.0:1769077051.889492:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.889499:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077051.889530:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077051.889550:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913399040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077051.889567:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913398912:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077051.889583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.889601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077051.889605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.889614:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2757 00000100:00080000:0.0:1769077051.889619:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.889624:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077051.889632:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077051.889650:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913399168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077051.889659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.889663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077051.889666:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.889670:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2757 00000100:00080000:0.0:1769077051.889673:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.889678:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077051.889683:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077051.889696:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913399296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077051.889706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.889711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077051.889714:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077051.889719:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399040:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077051.889721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2757 00000100:00080000:0.0:1769077051.889730:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.889736:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077051.889741:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077051.889757:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913399424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077051.889766:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399168:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077051.889772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.889776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077051.889778:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.889786:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2757 00000100:00080000:0.0:1769077051.889791:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.889796:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077051.889801:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077051.889817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913399552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077051.889822:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399296:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077051.889851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.889856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077051.889859:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.889864:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2757 00000100:00080000:0.0:1769077051.889869:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.889873:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077051.889877:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077051.889893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913399680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077051.889897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913399424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077051.889950:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913399552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077051.889971:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913399680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077052.081201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077052.593181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077052.593183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077052.593198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077052.593275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077052.913267:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077053.105212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077053.617160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077053.617160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077053.617191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077053.617284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077053.937247:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077054.129198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077054.641158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077054.641158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077054.641158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077054.641210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077054.961371:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769077054.961539:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769077054.961586:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077054.961606:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077054.961654:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077054.961664:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077054.961675:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077054.961688:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077054.961700:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077054.961705:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077054] req@ffff9350103f9c00 x1855008913399040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1769077054.961712:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077054.961745:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913399040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077054.961759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077054.961766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077054.961770:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077054.961780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077054] req@ffff9350103f9180 x1855008913399168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077054.961793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913399168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077054.961802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077054.961806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077054.961811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077054.961820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077054] req@ffff9350103fb100 x1855008913399296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077054.961834:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913399296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077054.961843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077054.961847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077054.961851:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077054.961859:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077054] req@ffff9350103faa00 x1855008913398912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077054.961875:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913398912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077054.961884:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077054.961887:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077054.961892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913398912:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077054.961929:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077054] req@ffff9350103fb480 x1855008913399680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077054.961944:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913399680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077054.961954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077054.961957:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077054.961961:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913399680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077054.961969:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077054] req@ffff9350103f8380 x1855008913399424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077054.961984:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913399424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077054.961994:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077054.961998:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077054.962001:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913399424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077054.962009:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077054] req@ffff9350103f8700 x1855008913399552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077054.962028:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913399552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077054.962038:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077054.962041:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077054.962045:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913399552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077055.153162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077055.665251:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077055.665255:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077055.665293:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077055.665423:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077055.985301:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077056.177195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077056.689126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077056.689126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077056.689172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077056.689180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077057.009215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077057.009291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077057.009303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077057.009308:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077057.009321:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2762 00000100:00080000:1.0:1769077057.009335:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077057.009345:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077057.009390:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913399808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077057.009449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077057.009453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077057.009456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077057.009462:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2762 00000100:00080000:1.0:1769077057.009466:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077057.009470:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077057.009482:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077057.009516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913399936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077057.009530:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913399808:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077057.009542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077057.009547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077057.009550:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077057.009556:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2762 00000100:00080000:1.0:1769077057.009561:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077057.009565:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077057.009572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077057.009586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913400064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077057.009596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077057.009600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077057.009603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077057.009607:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2762 00000100:00080000:1.0:1769077057.009612:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077057.009619:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077057.009620:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769077057.009625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077057.009641:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913400192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077057.009652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077057.009656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077057.009659:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077057.009664:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2762 00000100:00080000:1.0:1769077057.009669:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077057.009672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077057/real 1769077057] req@ffff9350100ca680 x1855008913399808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077057.009678:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077057.009696:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077057.009702:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913399808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077057.009722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769077057.009724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913400320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077057.009731:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077057.009742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913399808:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077057.009746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077057.009752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077057.009755:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077057.009760:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2762 00000100:00080000:1.0:1769077057.009765:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077057.009770:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077057.009776:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077057.009780:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077057.009793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913400448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077057.009806:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077057.009811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077057.009815:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077057.009816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077057.009825:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2762 00000100:00080000:1.0:1769077057.009831:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077057.009837:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077057.009843:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077057/real 1769077057] req@ffff9350100c9f80 x1855008913399936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077057.009844:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077057.009862:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913400576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077057.009867:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913399936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077057.009903:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077057.009909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077057.009914:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913399936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077057.009923:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077057.009949:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077057.009962:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077057/real 1769077057] req@ffff9350100cad80 x1855008913400064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077057.009975:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913400064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077057.009985:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077057.009988:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077057.009992:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077057.009999:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077057.010026:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077057.010037:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077057/real 1769077057] req@ffff9350100c8a80 x1855008913400192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077057.010075:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913400192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077057.010085:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077057.010089:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077057.010092:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077057.010100:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913400320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077057.010129:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077057.010142:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077057/real 1769077057] req@ffff9350100c8000 x1855008913400320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077057.010170:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913400320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077057.010181:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077057.010184:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077057.010188:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913400320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077057.010209:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913400448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077057.010231:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077057.010243:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077057/real 1769077057] req@ffff9350100cb100 x1855008913400448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077057.010263:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913400448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077057.010273:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077057.010277:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077057.010281:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913400448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077057.010287:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913400576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077057.010307:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077057.010319:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077057/real 1769077057] req@ffff9350100c9180 x1855008913400576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077057.010331:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913400576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077057.010351:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077057.010356:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077057.010360:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913400576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077057.201204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077057.713209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077057.713209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077057.713209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077057.713402:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077058.033309:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077058.225208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077058.737243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077058.737243:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077058.737247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077058.737295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077059.057274:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077059.249203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077059.761173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077059.761174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077059.761174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077059.761246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077060.081271:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077060.273150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077060.785260:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077060.785261:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077060.785272:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077060.785340:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077061.105202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077061.297240:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077061.809222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077061.809222:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077061.809222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077061.809342:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077062.129177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077062.129191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1769077062.129193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077062.129197:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077062.129208:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2767 00000100:00080000:1.0:1769077062.129221:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077062.129229:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077062.129275:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913400704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077062.129297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077062.129300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077062.129302:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077062.129305:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2767 00000100:00080000:1.0:1769077062.129308:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077062.129312:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077062.129323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077062.129333:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913400832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077062.129341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077062.129345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077062.129347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077062.129350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913400704:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077062.129351:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2767 00000100:00080000:1.0:1769077062.129356:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077062.129362:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077062.129366:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077062.129371:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913400960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077062.129377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077062.129380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077062.129382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077062.129384:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2767 00000100:00080000:1.0:1769077062.129386:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077062.129389:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077062.129392:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077062.129399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913401088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077062.129405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077062.129409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077062.129410:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077062.129413:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2767 00000100:00080000:1.0:1769077062.129424:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077062.129427:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077062.129430:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077062.129438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913401216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077062.129447:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077062.129448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077062.129454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077062.129455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077062.129458:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2767 00000100:00080000:1.0:1769077062.129461:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077062.129463:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077062.129466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077062.129474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913401344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077062.129481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077062.129483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077062.129484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077062.129487:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2767 00000100:00080000:1.0:1769077062.129490:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077062.129492:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077062.129497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077062.129504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913401472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077062.129512:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077062/real 1769077062] req@ffff9350100c9c00 x1855008913400704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077062.129537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913400704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077062.129550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077062.129556:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077062.129561:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913400704:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077062.129589:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077062.129616:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077062.129629:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077062/real 1769077062] req@ffff9350100c9180 x1855008913400832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077062.129643:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913400832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077062.129653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077062.129656:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077062.129660:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077062.129667:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077062.129687:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077062.129697:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077062/real 1769077062] req@ffff9350100cb100 x1855008913400960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077062.129710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913400960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077062.129719:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077062.129722:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077062.129726:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913400960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077062.129732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077062.129751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077062.129763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077062/real 1769077062] req@ffff9350100c8000 x1855008913401088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077062.129776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913401088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077062.129785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077062.129789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077062.129794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077062.129800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913401216:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077062.129821:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077062.129834:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077062/real 1769077062] req@ffff9350100c8a80 x1855008913401216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077062.129847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913401216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077062.129856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077062.129859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077062.129863:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913401216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077062.129869:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913401344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077062.129890:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077062.129902:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077062/real 1769077062] req@ffff9350100cad80 x1855008913401344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077062.129915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913401344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077062.129924:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077062.129927:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077062.129931:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913401344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077062.129937:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913401472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077062.129957:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077062.129970:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077062/real 1769077062] req@ffff9350100c9f80 x1855008913401472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077062.129983:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913401472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077062.129992:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077062.129995:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077062.129999:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913401472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077062.321169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077062.833208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077062.833208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077062.833230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077062.833251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077063.153246:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077063.345249:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077063.857200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077063.857200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077063.857200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077063.857245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077064.177183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077064.369157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077064.881197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077064.881206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077064.881240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077064.881247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077065.201176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077065.393170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077065.905184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077065.905187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077065.905215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077065.905330:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077066.225136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077066.417148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077066.929168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077066.929168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077066.929194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077066.929267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077067.249224:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077067.249235:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077067.249240:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077067.249251:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2772 00000100:00100000:3.0:1769077067.249260:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077067.249264:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077067.249275:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077067.249320:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913401600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077067.249349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077067.249351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077067.249352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077067.249356:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2772 00000100:00080000:1.0:1769077067.249358:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077067.249361:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077067.249375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077067.249391:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913401600:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077067.249395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913401728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077067.249413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077067.249417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077067.249420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077067.249427:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2772 00000100:00080000:1.0:1769077067.249431:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077067.249435:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077067.249440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077067.249451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913401856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077067.249459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077067.249462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077067.249470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077067.249471:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077067.249482:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2772 00000100:00080000:1.0:1769077067.249490:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077067.249494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077067.249500:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077067.249510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913401984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077067.249520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077067.249527:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077067/real 1769077067] req@ffff9350100ca680 x1855008913401600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077067.249528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077067.249536:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077067.249546:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2772 00000100:00080000:1.0:1769077067.249556:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077067.249564:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077067.249569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077067.249574:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913401600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077067.249581:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913402112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077067.249601:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077067.249604:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077067.249611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077067.249612:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077067.249614:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077067.249623:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913401600:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077067.249624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2772 00000100:00080000:1.0:1769077067.249635:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077067.249641:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077067.249647:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077067.249660:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913402240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077067.249661:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401728:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077067.249689:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077067.249692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077067.249698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077067.249699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077067.249712:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2772 00000100:00080000:1.0:1769077067.249717:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077067.249724:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077067.249726:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077067/real 1769077067] req@ffff9350100c8e00 x1855008913401728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077067.249734:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077067.249754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913402368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077067.249762:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913401728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077067.249794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077067.249797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077067.249801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077067.249808:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077067.249824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077067.249831:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077067/real 1769077067] req@ffff9350100c9880 x1855008913401856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077067.249839:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913401856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077067.249844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077067.249847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077067.249850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077067.249856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077067.249876:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077067.249887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077067/real 1769077067] req@ffff9350100c9500 x1855008913401984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077067.249899:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913401984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077067.249914:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077067.249917:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077067.249919:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913401984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077067.249924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913402112:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077067.249940:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077067.249948:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077067/real 1769077067] req@ffff9350100caa00 x1855008913402112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077067.249956:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913402112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077067.249962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077067.249964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077067.249967:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913402112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077067.250005:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913402240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077067.250022:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077067.250030:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077067/real 1769077067] req@ffff9350100c8380 x1855008913402240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077067.250038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913402240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077067.250044:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077067.250046:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077067.250074:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913402240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077067.250078:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913402368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077067.250090:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077067.250098:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077067/real 1769077067] req@ffff9350100ca300 x1855008913402368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077067.250106:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913402368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077067.250111:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077067.250113:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077067.250115:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913402368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077067.441179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077067.953170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077067.953170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077067.953170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077067.953298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077068.273193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077068.465252:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077068.977160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077068.977169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077068.977187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077068.977209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077069.297162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077069.489116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077070.001153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077070.001153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077070.001157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077070.001333:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077070.321144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077070.513273:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077071.025140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077071.025157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077071.025183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077071.025184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077071.345212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077071.537157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077072.049173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077072.049173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077072.049174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077072.049245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077072.369137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077072.369152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077072.369161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077072.369166:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077072.369179:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2778 00000100:00080000:3.0:1769077072.369191:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077072.369199:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077072.369249:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913402496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077072.369274:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077072.369277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077072.369279:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077072.369282:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2778 00000100:00080000:3.0:1769077072.369285:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077072.369290:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077072.369301:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077072.369312:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913402496:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077072.369315:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913402624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077072.369326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077072.369329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077072.369330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077072.369334:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2778 00000100:00080000:3.0:1769077072.369337:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077072.369339:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077072.369343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077072.369352:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913402752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077072.369359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077072.369362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077072.369363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077072.369366:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2778 00000100:00080000:3.0:1769077072.369368:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077072.369371:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077072.369375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077072.369383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913402880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077072.369390:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077072.369395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077072.369399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077072.369401:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077072.369405:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2778 00000100:00080000:3.0:1769077072.369409:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077072.369412:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077072.369415:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077072.369424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913403008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077072.369431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077072.369434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077072.369436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077072.369439:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077072/real 1769077072] req@ffff935002a7f100 x1855008913402496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077072.369440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2778 00000100:00080000:3.0:1769077072.369445:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077072.369451:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077072.369457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077072.369471:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913402496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077072.369471:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913403136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077072.369503:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077072.369505:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077072.369506:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077072.369510:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077072.369515:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077072.369517:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2778 00000100:00080000:3.0:1769077072.369520:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077072.369523:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077072.369526:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077072.369533:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913402496:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077072.369537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913403264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077072.369565:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913402624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077072.369583:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077072.369593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077072/real 1769077072] req@ffff935002a7c380 x1855008913402624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077072.369601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913402624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077072.369608:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077072.369610:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077072.369612:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913402624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077072.369616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913402752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077072.369628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077072.369634:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077072/real 1769077072] req@ffff935002a7ca80 x1855008913402752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077072.369641:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913402752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077072.369646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077072.369648:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077072.369650:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913402752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077072.369655:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913402880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077072.369673:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077072.369685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077072/real 1769077072] req@ffff935002a7e300 x1855008913402880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077072.369697:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913402880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077072.369707:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077072.369710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077072.369713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913402880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077072.369720:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403008:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077072.369736:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077072.369744:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077072/real 1769077072] req@ffff935002a7c700 x1855008913403008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077072.369752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913403008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077072.369758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077072.369760:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077072.369762:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077072.369766:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077072.369776:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077072.369781:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077072/real 1769077072] req@ffff935002a7f800 x1855008913403136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077072.369793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913403136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077072.369799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077072.369800:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077072.369802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077072.369805:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077072.369815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077072.369820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077072/real 1769077072] req@ffff935002a7c000 x1855008913403264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077072.369827:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913403264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077072.369833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077072.369835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077072.369837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077072.561198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077073.073140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077073.073141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077073.073142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077073.073183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077073.393169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077073.585159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077074.097176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077074.097176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077074.097178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077074.097228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077074.417170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077074.609173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077075.121174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077075.121199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077075.121203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077075.121238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077075.441180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077075.633142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077076.145181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077076.145182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077076.145191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077076.145245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077076.465217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077076.657175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077077.169106:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077077.169134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077077.169163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077077.169248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077077.489166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077077.489192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077077.489197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077077.489200:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077077.489207:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2783 00000100:00080000:1.0:1769077077.489217:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077077.489223:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077077.489258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913403392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077077.489288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077077.489291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077077.489293:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077077.489301:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2783 00000100:00080000:1.0:1769077077.489304:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077077.489308:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077077.489316:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077077.489329:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913403520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077077.489337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077077.489340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077077.489342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077077.489345:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2783 00000100:00080000:1.0:1769077077.489348:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077077.489350:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077077.489370:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077077.489379:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913403392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077077.489386:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913403648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077077.489398:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077077.489402:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077077.489404:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077077.489408:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2783 00000100:00080000:1.0:1769077077.489414:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077077.489427:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077077.489433:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077077.489445:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913403776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077077.489452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077077.489455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077077.489457:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077077.489460:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2783 00000100:00080000:1.0:1769077077.489464:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077077.489467:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077077.489470:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077077.489477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913403904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077077.489483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077077.489486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077077.489487:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077077.489492:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077077.489496:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2783 00000100:00080000:1.0:1769077077.489500:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077077.489512:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077077.489515:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077077.489523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913404032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077077.489529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077077.489532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077077.489533:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077077.489535:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2783 00000100:00080000:1.0:1769077077.489537:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077077.489539:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077077.489542:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077077.489548:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913404160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077077.489606:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077077/real 1769077077] req@ffff9350100cb800 x1855008913403392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077077.489637:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913403392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077077.489658:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077077.489663:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077077.489667:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913403392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077077.489693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913403520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077077.489716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077077.489726:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077077/real 1769077077] req@ffff9350100ca300 x1855008913403520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077077.489736:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913403520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077077.489742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077077.489745:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077077.489747:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913403520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077077.489767:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913403648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077077.489779:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077077.489785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077077/real 1769077077] req@ffff9350100c8380 x1855008913403648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077077.489793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913403648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077077.489803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077077.489805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077077.489870:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913403648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077077.489879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913403776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077077.489901:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077077.489912:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077077/real 1769077077] req@ffff9350100caa00 x1855008913403776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077077.489922:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913403776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077077.489929:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077077.489932:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077077.489935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913403776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077077.489939:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403904:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077077.489951:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077077.489958:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077077/real 1769077077] req@ffff9350100c9500 x1855008913403904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077077.489966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913403904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077077.489972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077077.489974:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077077.489977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913403904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077077.489981:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077077.489991:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077077.489997:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077077/real 1769077077] req@ffff9350100c9880 x1855008913404032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077077.490005:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913404032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077077.490010:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077077.490012:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077077.490014:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077077.490017:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077077.490026:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077077.490032:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077077/real 1769077077] req@ffff9350100c8e00 x1855008913404160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077077.490040:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913404160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077077.490045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077077.490047:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077077.490072:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077077.681171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077078.193201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077078.193203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077078.193213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077078.193273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077078.513273:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077078.705166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077079.217147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077079.217157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077079.217200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077079.217296:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077079.537123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077079.729123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077080.241155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077080.241165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077080.241209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077080.241227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077080.561189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077080.753196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077081.265146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077081.265146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077081.265146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077081.265173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077081.585164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077081.777123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077082.289156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077082.289157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077082.289165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077082.289205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077082.609159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077082.609167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077082.609175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077082.609179:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077082.609189:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2788 00000100:00080000:3.0:1769077082.609199:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077082.609207:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077082.609247:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913404288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077082.609274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077082.609279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077082.609281:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077082.609286:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2788 00000100:00080000:3.0:1769077082.609292:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077082.609297:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077082.609315:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913404288:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077082.609320:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077082.609334:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913404416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077082.609344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077082.609347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077082.609348:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077082.609352:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2788 00000100:00080000:3.0:1769077082.609355:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077082.609358:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077082.609363:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077082.609371:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913404544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077082.609378:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077082.609381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077082.609382:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077082.609385:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2788 00000100:00080000:3.0:1769077082.609387:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077082.609390:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077082.609393:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077082.609398:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077082.609404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913404672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077082.609412:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077082.609414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077082.609417:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077082.609420:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2788 00000100:00080000:3.0:1769077082.609425:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077082.609427:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077082.609430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077082.609441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913404800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077082.609448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077082.609452:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077082.609455:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077082/real 1769077082] req@ffff935002a7df80 x1855008913404288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077082.609455:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077082.609462:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2788 00000100:00080000:3.0:1769077082.609465:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077082.609471:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077082.609477:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077082.609487:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913404928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077082.609492:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913404288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077082.609497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077082.609505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077082.609508:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077082.609515:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2788 00000100:00080000:0.0:1769077082.609518:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077082.609519:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077082.609525:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077082.609529:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769077082.609531:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077082.609537:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913404288:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077082.609542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913405056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077082.609570:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913404416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077082.609594:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077082.609607:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077082/real 1769077082] req@ffff935002a7d180 x1855008913404416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077082.609621:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913404416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077082.609631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077082.609634:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077082.609638:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913404416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077082.609646:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913404544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077082.609663:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077082.609672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077082/real 1769077082] req@ffff935002a7dc00 x1855008913404544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077082.609685:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913404544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077082.609694:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077082.609698:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077082.609702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913404544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077082.609709:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913404672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077082.609746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077082.609758:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077082/real 1769077082] req@ffff935002a7fb80 x1855008913404672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077082.609771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913404672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077082.609780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077082.609783:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077082.609787:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913404672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077082.609793:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404800:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077082.609810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077082.609820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077082/real 1769077082] req@ffff935002a7c000 x1855008913404800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077082.609833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913404800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077082.609841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077082.609844:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077082.609848:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077082.609854:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077082.609869:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077082.609878:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077082/real 1769077082] req@ffff935002a7f800 x1855008913404928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077082.609890:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913404928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077082.609899:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077082.609902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077082.609905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913404928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077082.609911:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077082.609926:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077082.609935:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077082/real 1769077082] req@ffff935002a7c700 x1855008913405056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077082.609946:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913405056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077082.609961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077082.609964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077082.609968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077082.801179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077083.313187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077083.313187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077083.313187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077083.313227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077083.633159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077083.825152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077084.337147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077084.337148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077084.337148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077084.337195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077084.657193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077084.849182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077085.361164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077085.361164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077085.361174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077085.361231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077085.681134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077085.873225:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077086.385145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077086.385146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077086.385170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077086.385212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077086.705126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077086.897139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077087.409154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077087.409154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077087.409154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077087.409185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077087.729117:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077087.729118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077087.729127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077087.729131:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077087.729139:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2793 00000100:00080000:3.0:1769077087.729159:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077087.729167:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077087.729224:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913405184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077087.729267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077087.729271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077087.729273:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077087.729278:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2793 00000100:00080000:3.0:1769077087.729283:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077087.729287:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077087.729298:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077087.729315:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913405312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077087.729322:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913405184:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077087.729329:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077087.729333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077087.729335:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077087.729339:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2793 00000100:00080000:3.0:1769077087.729343:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077087.729346:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077087.729351:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077087.729361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913405440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077087.729375:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077087.729378:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077087.729379:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077087.729382:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2793 00000100:00080000:3.0:1769077087.729385:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077087.729387:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077087.729390:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077087.729397:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077087.729401:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913405568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077087.729409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077087.729412:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077087.729414:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077087.729419:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2793 00000100:00080000:3.0:1769077087.729422:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077087.729426:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077087.729430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077087.729444:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077087/real 1769077087] req@ffff935002a7e300 x1855008913405184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077087.729444:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913405696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077087.729467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077087.729469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077087.729471:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077087.729473:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2793 00000100:00080000:3.0:1769077087.729474:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077087.729479:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077087.729482:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077087.729492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913405824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077087.729498:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913405184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077087.729500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077087.729507:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077087.729510:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077087.729518:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2793 00000100:00080000:3.0:1769077087.729522:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077087.729523:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077087.729525:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077087.729531:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077087.729533:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077087.729541:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913405952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077087.729542:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913405184:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077087.729576:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913405312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077087.729598:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077087.729611:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077087/real 1769077087] req@ffff935002a7ca80 x1855008913405312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077087.729625:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913405312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077087.729665:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077087.729670:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077087.729674:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913405312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077087.729681:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913405440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077087.729701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077087.729712:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077087/real 1769077087] req@ffff935002a7c380 x1855008913405440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077087.729725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913405440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077087.729734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077087.729738:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077087.729742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913405440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077087.729747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913405568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077087.729763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077087.729773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077087/real 1769077087] req@ffff935002a7f100 x1855008913405568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077087.729785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913405568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077087.729794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077087.729797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077087.729801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913405568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077087.729807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405696:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077087.729822:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077087.729831:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077087/real 1769077087] req@ffff935002a7c700 x1855008913405696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077087.729844:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913405696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077087.729852:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077087.729855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077087.729859:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405696:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077087.729865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077087.729881:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077087.729890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077087/real 1769077087] req@ffff935002a7f800 x1855008913405824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077087.729901:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913405824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077087.729910:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077087.729913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077087.729916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077087.729922:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077087.729936:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077087.729947:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077087/real 1769077087] req@ffff935002a7c000 x1855008913405952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077087.729959:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913405952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077087.729968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077087.729971:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077087.729974:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913405952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077087.921145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077088.433123:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077088.433130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077088.433136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077088.433178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077088.753161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077088.945166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077089.457146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077089.457150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077089.457164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077089.457173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077089.777104:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077089.969156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077090.481116:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077090.481117:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077090.481132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077090.481167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077090.801125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077090.993129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077091.505137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077091.505137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077091.505138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077091.505168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077091.825184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077092.017088:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077092.529109:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077092.529122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077092.529122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077092.529162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077092.849140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077092.849141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077092.849150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077092.849154:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077092.849162:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2798 00000100:00080000:2.0:1769077092.849173:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077092.849181:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077092.849216:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913406080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077092.849243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077092.849247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077092.849249:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077092.849254:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2798 00000100:00080000:2.0:1769077092.849258:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077092.849262:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077092.849272:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077092.849286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913406208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077092.849296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077092.849300:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077092.849301:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077092.849313:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2798 00000100:00080000:2.0:1769077092.849317:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077092.849321:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077092.849326:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077092.849339:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913406336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077092.849350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077092.849354:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077092.849356:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077092.849364:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2798 00000100:00100000:0.0:1769077092.849367:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913406080:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077092.849368:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077092.849375:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077092.849382:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077092.849396:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913406464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077092.849404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077092.849407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077092.849408:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077092.849411:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2798 00000100:00080000:2.0:1769077092.849414:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077092.849417:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077092.849420:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077092.849430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913406592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077092.849436:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077092.849440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077092.849443:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077092.849446:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2798 00000100:00080000:2.0:1769077092.849449:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077092.849452:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077092.849459:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769077092.849463:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077092.849477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913406720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077092.849487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077092.849490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077092.849491:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077092.849493:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2798 00000100:00080000:2.0:1769077092.849496:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077092.849499:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077092.849504:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077092.849516:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077092/real 1769077092] req@ffff935002a61c00 x1855008913406080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077092.849517:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913406848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077092.849582:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913406080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.849593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.849598:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077092.849602:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913406080:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077092.849610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913406208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077092.849628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077092.849638:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077092/real 1769077092] req@ffff935002a61f80 x1855008913406208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.849646:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913406208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.849652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.849655:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077092.849657:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913406208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077092.849682:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913406336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077092.849693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077092.849699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077092/real 1769077092] req@ffff935002a62680 x1855008913406336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.849707:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913406336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.849712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.849714:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077092.849716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913406336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077092.849720:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913406464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077092.849730:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077092.849736:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077092/real 1769077092] req@ffff935002a60a80 x1855008913406464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.849744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913406464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.849749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.849751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077092.849753:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913406464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077092.849756:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913406592:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077092.849766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077092.849773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077092/real 1769077092] req@ffff935002a63100 x1855008913406592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.849781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913406592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.849786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.849788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077092.849790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913406592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077092.849793:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913406720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077092.849802:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077092.849808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077092/real 1769077092] req@ffff935002a61180 x1855008913406720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.849815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913406720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.849824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.849827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077092.849829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913406720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077092.849833:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913406848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077092.849842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077092.849847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077092/real 1769077092] req@ffff935002a61880 x1855008913406848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.849854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913406848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.849859:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.849861:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077092.849863:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913406848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077093.041117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077093.553134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077093.553159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077093.553179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077093.553199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077093.873173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077094.065139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077094.577119:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077094.577125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077094.577153:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077094.577164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077094.897170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077095.089148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077095.601173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077095.601193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077095.601309:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077095.601371:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077095.921159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077096.113142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077096.625129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077096.625178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077096.625185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077096.625197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077096.945152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077097.137124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077097.649128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077097.649137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077097.649171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077097.649212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077097.969165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077097.969182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077097.969191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077097.969203:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077097.969214:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2803 00000100:00080000:3.0:1769077097.969226:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077097.969233:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077097.969275:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913406976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077097.969306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077097.969310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077097.969313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077097.969318:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2803 00000100:00080000:3.0:1769077097.969322:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077097.969327:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077097.969338:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077097.969349:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913406976:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077097.969356:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913407104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077097.969371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077097.969376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077097.969379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077097.969384:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2803 00000100:00080000:3.0:1769077097.969388:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077097.969393:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077097.969398:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077097.969412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913407232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077097.969415:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077097.969428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077097.969433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077097.969435:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077097.969441:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2803 00000100:00080000:3.0:1769077097.969446:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077097.969450:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077097.969456:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff935002a7fb80 x1855008913406976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077097.969457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077097.969499:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913406976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.969499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913407360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077097.969536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077097.969538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077097.969542:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077097.969548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769077097.969549:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913406976:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077097.969552:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077097.969559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2803 00000100:00080000:3.0:1769077097.969564:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077097.969570:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077097.969575:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913407104:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077097.969580:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077097.969598:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913407488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077097.969602:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077097.969618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077097.969619:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff935002a7dc00 x1855008913407104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077097.969625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077097.969628:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077097.969634:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2803 00000100:00080000:3.0:1769077097.969641:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077097.969647:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077097.969656:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077097.969665:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913407104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.969672:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913407616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077097.969679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077097.969687:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077097.969690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077097.969693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913407104:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077097.969699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077097.969703:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913407232:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077097.969704:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077097.969712:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2803 00000100:00080000:3.0:1769077097.969718:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077097.969723:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077097.969729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077097.969732:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077097.969740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff935002a7d180 x1855008913407232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.969749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913407744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077097.969752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913407232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077097.969785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077097.969788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077097.969790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913407232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077097.969795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913407360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077097.969807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077097.969813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff935002a7df80 x1855008913407360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077097.969820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913407360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077097.969825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077097.969827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077097.969829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913407360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077097.969841:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913407488:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077097.969851:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077097.969858:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff935002a7fb80 x1855008913407488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077097.969866:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913407488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077097.969871:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077097.969872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077097.969875:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913407488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077097.969879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913407616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077097.969888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077097.969893:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff935002a7c000 x1855008913407616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077097.969900:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913407616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077097.969905:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077097.969907:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077097.969909:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913407616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077097.969912:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913407744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077097.969921:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077097.969926:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff935002a7f800 x1855008913407744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077097.969933:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913407744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077097.969938:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077097.969940:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077097.969942:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913407744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077098.161171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077098.673136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077098.673136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077098.673164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077098.673192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077098.993150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077099.185173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077099.697135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077099.697139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077099.697142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077099.697177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077100.017217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077100.209167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077100.721182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077100.721209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077100.721241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077100.721242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077101.041186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077101.233181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077101.745139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077101.745146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077101.745182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077101.745245:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077102.065276:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077102.257200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077102.769191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077102.769191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077102.769191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077102.769221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077103.089180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077103.089193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077103.089194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077103.089201:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077103.089215:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2808 00000100:00080000:0.0:1769077103.089230:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077103.089241:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077103.089309:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913407872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077103.089356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077103.089362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077103.089365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077103.089370:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2808 00000100:00080000:0.0:1769077103.089414:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077103.089430:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077103.089440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077103.089452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913408000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077103.089462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077103.089465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077103.089467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077103.089471:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2808 00000100:00080000:0.0:1769077103.089474:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077103.089477:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077103.089493:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077103.089497:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913407872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077103.089505:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913408128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077103.089515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077103.089517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077103.089518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077103.089521:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2808 00000100:00080000:0.0:1769077103.089523:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077103.089526:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077103.089529:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077103.089538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913408256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077103.089546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077103.089549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077103.089550:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077103.089553:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2808 00000100:00080000:0.0:1769077103.089556:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077103.089558:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077103.089561:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077103.089571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913408384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077103.089578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077103.089582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077103.089586:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077103.089587:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077103.089592:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2808 00000100:00080000:0.0:1769077103.089596:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077103.089599:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077103.089603:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077103.089614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913408512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077103.089620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077103.089623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077103.089625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077103.089628:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2808 00000100:00080000:0.0:1769077103.089631:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077103.089634:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077103.089646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077103.089659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077103/real 1769077103] req@ffff9350103f8700 x1855008913407872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077103.089660:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913408640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077103.089703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913407872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077103.089743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077103.089750:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077103.089756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913407872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077103.089768:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408000:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077103.089800:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077103.089815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077103/real 1769077103] req@ffff9350103f8380 x1855008913408000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077103.089827:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913408000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077103.089836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077103.089840:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077103.089843:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077103.089873:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077103.089911:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077103.089928:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077103/real 1769077103] req@ffff9350103fb480 x1855008913408128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077103.089948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913408128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077103.089960:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077103.089964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077103.089967:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077103.089974:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077103.089994:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077103.090004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077103/real 1769077103] req@ffff9350103faa00 x1855008913408256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077103.090034:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913408256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077103.090044:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077103.090048:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077103.090051:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077103.090116:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913408384:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077103.090134:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077103.090143:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077103/real 1769077103] req@ffff9350103fb100 x1855008913408384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077103.090152:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913408384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077103.090176:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077103.090179:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077103.090182:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913408384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077103.090188:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913408512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077103.090203:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077103.090211:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077103/real 1769077103] req@ffff9350103f9180 x1855008913408512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077103.090222:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913408512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077103.090229:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077103.090232:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077103.090234:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913408512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077103.090239:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913408640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077103.090251:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077103.090259:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077103/real 1769077103] req@ffff9350103f9c00 x1855008913408640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077103.090268:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913408640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077103.090275:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077103.090278:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077103.090281:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913408640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077103.281137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077103.793117:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077103.793143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077103.793151:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077103.793307:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077104.113170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077104.305125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077104.817132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077104.817132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077104.817132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077104.817193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077105.137122:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077105.329235:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077105.841134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077105.841141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077105.841170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077105.841207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077106.161215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077106.353224:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077106.865192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077106.865192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077106.865192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077106.865258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077107.185170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077107.377188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077107.889136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077107.889136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077107.889169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077107.889175:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077108.209200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077108.209200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077108.209216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077108.209221:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077108.209231:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2813 00000100:00080000:0.0:1769077108.209243:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077108.209251:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077108.209289:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913408768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077108.209328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077108.209331:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077108.209333:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077108.209336:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2813 00000100:00080000:0.0:1769077108.209340:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077108.209344:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077108.209352:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077108.209360:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913408768:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077108.209366:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913408896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077108.209376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077108.209378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077108.209380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077108.209383:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2813 00000100:00080000:0.0:1769077108.209385:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077108.209387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077108.209391:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077108.209401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913409024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077108.209407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077108.209409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077108.209410:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077108.209413:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2813 00000100:00080000:0.0:1769077108.209416:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077108.209418:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077108.209426:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077108.209426:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077108.209440:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913409152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077108.209454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077108.209457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077108.209458:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077108.209462:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2813 00000100:00080000:0.0:1769077108.209465:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077108.209469:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077108/real 1769077108] req@ffff9350103f9f80 x1855008913408768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077108.209471:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077108.209477:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077108.209492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913409280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077108.209496:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913408768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077108.209515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077108.209518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077108.209520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077108.209520:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077108.209528:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077108.209529:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2813 00000100:00080000:0.0:1769077108.209534:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077108.209537:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913408768:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077108.209540:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077108.209546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077108.209555:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913409408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077108.209563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077108.209564:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408896:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077108.209569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077108.209572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077108.209576:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2813 00000100:00080000:0.0:1769077108.209578:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077108.209581:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077108.209585:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077108.209590:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077108.209597:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913409536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077108.209605:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077108/real 1769077108] req@ffff9350103fb800 x1855008913408896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077108.209616:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913408896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077108.209624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077108.209627:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077108.209629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913408896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077108.209634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077108.209645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077108.209651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077108/real 1769077108] req@ffff9350103fad80 x1855008913409024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077108.209659:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913409024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077108.209670:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077108.209674:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077108.209678:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077108.209683:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077108.209702:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077108.209713:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077108/real 1769077108] req@ffff9350103fbb80 x1855008913409152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077108.209726:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913409152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077108.209735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077108.209739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077108.209743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077108.209748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913409280:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077108.209764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077108.209771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077108/real 1769077108] req@ffff9350103f9500 x1855008913409280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077108.209779:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913409280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077108.209785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077108.209787:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077108.209790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913409280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077108.209802:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913409408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077108.209813:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077108.209819:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077108/real 1769077108] req@ffff9350103f9c00 x1855008913409408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077108.209827:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913409408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077108.209833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077108.209835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077108.209837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913409408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077108.209840:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913409536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077108.209849:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077108.209854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077108/real 1769077108] req@ffff9350103f9180 x1855008913409536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077108.209860:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913409536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077108.209865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077108.209867:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077108.209869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913409536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077108.401206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077108.913189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077108.913189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077108.913236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077108.913254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077109.233140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077109.425354:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077109.937205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077109.937214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077109.937235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077109.937315:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077110.257253:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077110.449151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077110.961216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077110.961216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077110.961236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077110.961329:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077111.281156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077111.473171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077111.985125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077111.985131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077111.985152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077111.985161:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077112.305124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077112.497129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077113.009199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077113.009199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077113.009226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077113.009290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077113.329153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077113.329154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077113.329164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077113.329169:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077113.329178:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2819 00000100:00080000:1.0:1769077113.329190:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077113.329198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077113.329251:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913409664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077113.329281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077113.329289:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077113.329292:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077113.329297:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2819 00000100:00080000:1.0:1769077113.329301:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077113.329308:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077113.329318:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077113.329326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913409664:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077113.329334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913409792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077113.329348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077113.329352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077113.329355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077113.329360:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2819 00000100:00080000:1.0:1769077113.329364:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077113.329368:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077113.329373:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077113.329387:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913409920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077113.329389:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077113.329402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077113.329407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077113.329409:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077113.329415:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2819 00000100:00080000:1.0:1769077113.329420:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077113.329424:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077113.329427:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077113/real 1769077113] req@ffff9350100ca680 x1855008913409664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077113.329431:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077113.329450:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913409664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077113.329452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913410048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077113.329470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077113.329477:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077113.329478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077113.329483:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913409664:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077113.329486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077113.329490:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077113.329497:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2819 00000100:00080000:1.0:1769077113.329501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077113.329506:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409792:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077113.329508:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077113.329517:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077113.329528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077113.329533:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913410176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077113.329540:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077113/real 1769077113] req@ffff9350100c9f80 x1855008913409792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077113.329561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077113.329563:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913409792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077113.329568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077113.329573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077113.329576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077113.329581:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2819 00000100:00080000:2.0:1769077113.329584:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077113.329589:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077113.329590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409792:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077113.329599:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077113.329600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409920:192.168.204.121@tcp:8:kworker.0 00000008:00100000:1.0:1769077113.329607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077113.329622:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077113.329624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913410304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077113.329633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077113/real 1769077113] req@ffff9350100cad80 x1855008913409920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077113.329644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077113.329654:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077113.329658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077113.329663:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913409920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077113.329668:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2819 00000100:00080000:1.0:1769077113.329679:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077113.329680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077113.329687:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077113.329690:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077113.329693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913409920:192.168.204.121@tcp:8:kworker.0 00000008:00100000:1.0:1769077113.329700:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077113.329701:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077113.329722:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913410432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077113.329722:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077113.329732:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077113/real 1769077113] req@ffff9350100c8a80 x1855008913410048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077113.329746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913410048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077113.329752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077113.329756:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077113.329758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077113.329768:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913410176:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077113.329778:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077113.329783:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077113/real 1769077113] req@ffff9350100c8000 x1855008913410176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077113.329791:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913410176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077113.329801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077113.329803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077113.329805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913410176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077113.329808:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913410304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077113.329816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077113.329821:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077113/real 1769077113] req@ffff9350100cb100 x1855008913410304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077113.329828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913410304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077113.329833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077113.329835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077113.329837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913410304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077113.329841:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913410432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077113.329849:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077113.329855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077113/real 1769077113] req@ffff9350100c9180 x1855008913410432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077113.329861:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913410432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077113.329866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077113.329867:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077113.329870:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913410432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077113.521163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077114.033131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077114.033131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077114.033134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077114.033153:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077114.353123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077114.545120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077115.057147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077115.057147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077115.057147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077115.057174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077115.377150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077115.569163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077116.081198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077116.081204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077116.081204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077116.081239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077116.401125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077116.593219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077117.105222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077117.105222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077117.105276:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077117.105303:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077117.425169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077117.617187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077118.129229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077118.129230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077118.129230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077118.129261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077118.449180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077118.449185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077118.449196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077118.449202:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077118.449214:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2824 00000100:00080000:1.0:1769077118.449228:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077118.449238:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077118.449284:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913410560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077118.449319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077118.449323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077118.449326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077118.449330:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2824 00000100:00080000:1.0:1769077118.449334:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077118.449341:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077118.449353:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077118.449369:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913410560:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077118.449371:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913410688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077118.449388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077118.449392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077118.449394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077118.449398:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2824 00000100:00080000:1.0:1769077118.449402:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077118.449405:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077118.449410:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077118.449423:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913410816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077118.449434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077118.449438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077118.449440:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077118.449444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2824 00000100:00080000:1.0:1769077118.449448:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077118.449452:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077118.449457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077118.449476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913410944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077118.449489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077118.449491:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410688:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077118.449499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077118.449502:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077118.449509:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2824 00000100:00080000:1.0:1769077118.449513:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077118.449518:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077118.449524:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077118.449536:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077118.449542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913411072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077118.449555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077118.449561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410944:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769077118.449562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077118.449567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077118.449574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2824 00000100:00080000:1.0:1769077118.449579:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077118.449583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411072:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077118.449586:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077118.449596:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077118.449612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913411200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077118.449632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077118.449636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077118.449639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077118.449643:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411200:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769077118.449645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2824 00000100:00080000:1.0:1769077118.449654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077118.449659:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077118.449667:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077118.449681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913411328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077118.449711:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077118.641155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077119.153205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077119.153206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077119.153212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077119.153276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077119.473206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077119.665137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077120.177198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077120.177200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077120.177242:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077120.177295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077120.497194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077120.689159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077121.201153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077121.201153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077121.201153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077121.201194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077121.521239:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077121.521460:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077121.521518:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077121.521545:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077121.521610:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077121.521637:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077121.521653:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077121.521668:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077121.521682:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077121.521697:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077121.521711:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077118/real 1769077121] req@ffff9350100cad80 x1855008913411200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077121.521759:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913411200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077121.521782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077121.521792:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077121.521803:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077121.521818:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077118/real 1769077121] req@ffff9350100c8000 x1855008913410944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077121.521843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913410944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077121.521857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077121.521881:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077121.521887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077121.521899:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077118/real 1769077121] req@ffff9350100c8a80 x1855008913411072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077121.521920:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913411072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077121.521935:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077121.521941:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077121.521946:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411072:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077121.521958:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077118/real 1769077121] req@ffff9350100c9180 x1855008913410688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077121.521980:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913410688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077121.521994:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077121.521999:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077121.522006:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077121.522017:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077118/real 1769077121] req@ffff9350100cb100 x1855008913410816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077121.522039:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913410816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077121.522052:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077121.522057:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077121.522064:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913410816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077121.522155:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077118/real 1769077121] req@ffff9350100c9c00 x1855008913410560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077121.522179:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913410560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077121.522191:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077121.522197:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077121.522203:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913410560:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077121.522252:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077118/real 1769077121] req@ffff9350100c9f80 x1855008913411328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077121.522296:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913411328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077121.522311:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077121.522317:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077121.522324:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077121.713187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077122.225226:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077122.225226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077122.225226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077122.225266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077122.545130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077122.737128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077123.249152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077123.249152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077123.249152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077123.249211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077123.569155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077123.569158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077123.569168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077123.569173:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077123.569183:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2829 00000100:00080000:1.0:1769077123.569194:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077123.569202:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077123.569238:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913411456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077123.569264:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077123.569267:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077123.569269:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077123.569273:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2829 00000100:00080000:1.0:1769077123.569275:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077123.569280:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077123.569289:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077123.569298:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913411584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077123.569307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077123.569309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077123.569310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077123.569315:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913411456:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077123.569316:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2829 00000100:00080000:1.0:1769077123.569322:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077123.569328:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077123.569333:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077123.569346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913411712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077123.569356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077123.569360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077123.569362:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077123.569367:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2829 00000100:00080000:1.0:1769077123.569371:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077123.569376:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077123.569383:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077123.569409:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077123.569418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913411840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077123.569430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077123.569435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077123.569437:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077123.569443:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2829 00000100:00080000:1.0:1769077123.569447:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077123.569452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077123.569459:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077123.569467:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077123/real 1769077123] req@ffff9350100ca680 x1855008913411456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077123.569474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913411968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077123.569495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077123.569499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077123.569501:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077123.569507:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2829 00000100:00100000:2.0:1769077123.569510:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913411456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077123.569512:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077123.569522:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077123.569531:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077123.569533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077123.569543:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769077123.569551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913412096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077123.569551:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913411456:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077123.569568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077123.569573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077123.569575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077123.569581:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2829 00000100:00080000:1.0:1769077123.569585:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077123.569591:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077123.569597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913411584:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769077123.569599:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077123.569618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913412224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077123.569634:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077123.569654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077123/real 1769077123] req@ffff9350100c8e00 x1855008913411584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077123.569667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913411584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077123.569676:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077123.569679:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077123.569683:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913411584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077123.569688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913411712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077123.569712:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077123.569726:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077123/real 1769077123] req@ffff9350100c9880 x1855008913411712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077123.569737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913411712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077123.569745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077123.569748:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077123.569752:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913411712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077123.569757:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913411840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077123.569775:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077123.569786:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077123/real 1769077123] req@ffff9350100c9500 x1855008913411840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077123.569797:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913411840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077123.569805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077123.569808:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077123.569811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913411840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077123.569816:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411968:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077123.569833:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077123.569843:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077123/real 1769077123] req@ffff9350100caa00 x1855008913411968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077123.569860:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913411968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077123.569868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077123.569871:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077123.569874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913411968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077123.569880:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077123.569899:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077123.569910:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077123/real 1769077123] req@ffff9350100c8380 x1855008913412096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077123.569921:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913412096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077123.569929:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077123.569932:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077123.569935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077123.569956:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077123.569974:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077123.569986:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077123/real 1769077123] req@ffff9350100ca300 x1855008913412224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077123.569998:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913412224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077123.570012:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077123.570015:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077123.570019:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077123.761190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077124.273197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077124.273197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077124.273197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077124.273237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077124.593135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077124.785267:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077125.297132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077125.297148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077125.297196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077125.297197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077125.617151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077125.809147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077126.321252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077126.321254:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077126.321417:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077126.321467:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077126.641171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077126.833175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077127.345167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077127.345172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077127.345172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077127.345295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077127.665151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077127.857155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077128.369156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077128.369156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077128.369176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077128.369198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077128.689120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077128.689123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077128.689132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077128.689136:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077128.689145:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2834 00000100:00080000:1.0:1769077128.689157:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077128.689164:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077128.689200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913412352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077128.689223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077128.689226:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077128.689227:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077128.689230:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2834 00000100:00080000:1.0:1769077128.689233:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077128.689238:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077128.689246:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077128.689259:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913412352:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077128.689260:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913412480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077128.689274:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077128.689276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077128.689277:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077128.689280:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2834 00000100:00080000:1.0:1769077128.689282:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077128.689284:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077128.689288:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077128.689298:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913412608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077128.689308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077128.689312:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077128.689314:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077128.689318:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2834 00000100:00080000:1.0:1769077128.689322:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077128.689326:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077128.689330:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769077128.689334:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077128.689351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913412736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077128.689362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077128.689365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077128.689367:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077128.689372:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2834 00000100:00100000:2.0:1769077128.689372:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077128/real 1769077128] req@ffff9350100cb800 x1855008913412352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077128.689377:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077128.689385:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077128.689391:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077128.689395:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913412352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077128.689403:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913412864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077128.689404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077128.689413:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077128.689421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077128.689423:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913412352:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077128.689428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077128.689431:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077128.689435:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2834 00000100:00080000:1.0:1769077128.689438:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077128.689440:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077128.689443:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077128.689449:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913412480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077128.689454:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913412992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077128.689462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077128.689465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077128.689466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077128.689470:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2834 00000800:00000100:2.0:1769077128.689475:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077128.689476:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077128.689481:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077128.689485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077128.689490:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077128/real 1769077128] req@ffff9350100ca300 x1855008913412480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077128.689496:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913413120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077128.689506:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913412480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077128.689514:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077128.689519:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077128.689522:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913412480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077128.689526:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913412608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077128.689538:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077128.689544:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077128/real 1769077128] req@ffff9350100c8380 x1855008913412608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077128.689551:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913412608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077128.689557:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077128.689559:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077128.689561:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913412608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077128.689565:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913412736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077128.689575:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077128.689580:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077128/real 1769077128] req@ffff9350100caa00 x1855008913412736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077128.689588:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913412736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077128.689593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077128.689595:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077128.689597:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913412736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077128.689600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077128.689610:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077128.689615:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077128/real 1769077128] req@ffff9350100c9500 x1855008913412864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077128.689623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913412864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077128.689628:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077128.689630:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077128.689632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077128.689642:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077128.689651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077128.689657:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077128/real 1769077128] req@ffff9350100c9880 x1855008913412992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077128.689665:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913412992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077128.689670:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077128.689672:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077128.689673:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913412992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077128.689677:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913413120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077128.689686:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077128.689692:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077128/real 1769077128] req@ffff9350100c8e00 x1855008913413120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077128.689704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913413120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077128.689709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077128.689711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077128.689713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913413120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077128.881140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077129.393126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077129.393128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077129.393155:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077129.393170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077129.713154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077129.905157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077130.417136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077130.417147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077130.417164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077130.417165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077130.737155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077130.929151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077131.441124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077131.441170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077131.441648:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077131.441678:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077131.761151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077131.953137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077132.465192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077132.465263:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077132.465339:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077132.465359:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077132.785158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077132.977182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077133.489205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077133.489223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077133.489228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077133.489263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077133.809136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077133.809141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077133.809150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077133.809157:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077133.809167:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2839 00000100:00080000:0.0:1769077133.809179:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077133.809187:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077133.809221:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913413248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077133.809280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077133.809282:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913413248:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077133.809295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077133.809297:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077133.809302:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2839 00000100:00080000:0.0:1769077133.809306:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077133.809309:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077133.809318:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077133.809329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913413376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077133.809344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077133.809345:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077133.809350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077133.809353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077133.809357:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2839 00000100:00080000:0.0:1769077133.809362:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077133.809365:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077133.809369:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077133.809378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913413504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077133.809387:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077133/real 1769077133] req@ffff9350103fb100 x1855008913413248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077133.809388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077133.809396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077133.809399:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077133.809407:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2839 00000100:00080000:0.0:1769077133.809411:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077133.809414:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077133.809418:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913413248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077133.809419:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1769077133.809433:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077133.809434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913413632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077133.809441:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077133.809445:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913413248:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077133.809456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077133.809459:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077133.809460:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077133.809464:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2839 00000100:00080000:0.0:1769077133.809466:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077133.809468:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077133.809470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077133.809478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913413760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077133.809485:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913413376:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077133.809501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077133.809504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077133.809506:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077133.809510:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2839 00000100:00080000:0.0:1769077133.809514:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077133.809514:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077133.809523:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077133.809529:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077133.809530:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077133/real 1769077133] req@ffff9350103faa00 x1855008913413376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077133.809542:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913413376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077133.809555:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913413888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077133.809556:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077133.809564:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077133.809571:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913413376:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077133.809581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077133.809581:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913413504:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077133.809590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077133.809593:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077133.809597:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2839 00000100:00080000:0.0:1769077133.809602:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077133.809606:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077133.809609:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077133.809616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077133.809626:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077133/real 1769077133] req@ffff9350103fb480 x1855008913413504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077133.809637:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913414016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077133.809640:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913413504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077133.809663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077133.809667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077133.809669:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913413504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077133.809675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913413632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077133.809692:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077133.809704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077133/real 1769077133] req@ffff9350103f8380 x1855008913413632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077133.809717:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913413632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077133.809726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077133.809730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077133.809734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913413632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077133.809748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913413760:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077133.809763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077133.809770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077133/real 1769077133] req@ffff9350103f8700 x1855008913413760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077133.809778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913413760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077133.809784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077133.809786:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077133.809788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913413760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077133.809792:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913413888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077133.809801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077133.809807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077133/real 1769077133] req@ffff9350103fb100 x1855008913413888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077133.809815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913413888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077133.809820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077133.809821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077133.809823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913413888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077133.809826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077133.809834:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077133.809840:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077133/real 1769077133] req@ffff9350103f9180 x1855008913414016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077133.809847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913414016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077133.809853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077133.809854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077133.809856:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077134.001152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077134.513162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077134.513162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077134.513162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077134.513196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077134.833163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077135.025113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077135.537172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077135.537173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077135.537173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077135.537210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077135.857157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077136.049116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077136.561225:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077136.561226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077136.561231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077136.561276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077136.881166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077137.073147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077137.585174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077137.585196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077137.585209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077137.585242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077137.905167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077138.097125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077138.609161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077138.609161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077138.609213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077138.609277:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077138.929179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077138.929218:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077138.929225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077138.929229:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077138.929238:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2844 00000100:00080000:3.0:1769077138.929248:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077138.929255:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077138.929288:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913414144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077138.929316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077138.929319:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077138.929320:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077138.929324:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2844 00000100:00080000:3.0:1769077138.929327:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077138.929330:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077138.929338:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077138.929345:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913414144:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077138.929351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913414272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077138.929360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077138.929362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077138.929363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077138.929366:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2844 00000100:00080000:3.0:1769077138.929368:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077138.929370:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077138.929373:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077138.929382:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913414400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077138.929389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077138.929391:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077138.929392:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077138.929394:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2844 00000100:00080000:3.0:1769077138.929408:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077138.929411:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077138.929413:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077138.929421:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077138.929424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913414528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077138.929432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077138.929435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077138.929436:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077138.929440:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2844 00000100:00080000:3.0:1769077138.929444:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077138.929447:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077138.929450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077138.929457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913414656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077138.929464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077138.929466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077138.929468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077138.929471:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2844 00000100:00100000:2.0:1769077138.929475:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff935002a7c700 x1855008913414144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077138.929476:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077138.929482:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077138.929489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077138.929503:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913414144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077138.929504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913414784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077138.929530:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077138.929539:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077138.929541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077138.929546:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913414144:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077138.929547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077138.929551:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077138.929556:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2844 00000100:00080000:3.0:1769077138.929559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077138.929561:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077138.929564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077138.929571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913414912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077138.929575:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913414272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077138.929601:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077138.929611:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff935002a7f100 x1855008913414272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077138.929620:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913414272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077138.929631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077138.929633:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077138.929636:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913414272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077138.929640:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913414400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077138.929651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077138.929657:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff935002a7c380 x1855008913414400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077138.929665:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913414400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077138.929671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077138.929672:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077138.929675:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913414400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077138.929679:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913414528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077138.929697:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077138.929709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff935002a7ca80 x1855008913414528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077138.929722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913414528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077138.929731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077138.929735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077138.929739:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913414528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077138.929745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414656:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077138.929762:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077138.929769:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff935002a7e300 x1855008913414656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077138.929811:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913414656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077138.929817:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077138.929819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077138.929822:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077138.929826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077138.929842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077138.929849:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff935002a7f800 x1855008913414784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077138.929857:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913414784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077138.929862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077138.929864:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077138.929867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077138.929882:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077138.929892:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077138.929899:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff935002a7c000 x1855008913414912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077138.929907:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913414912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077138.929912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077138.929914:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077138.929916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913414912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077139.121161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077139.633100:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077139.633127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077139.633143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077139.633156:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077139.953128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077140.145117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077140.657116:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077140.657132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077140.657139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077140.657174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077140.977171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077141.169165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077141.681156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077141.681181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077141.681183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077141.681316:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077142.001137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077142.193113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077142.705128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077142.705129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077142.705148:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077142.705155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077143.025153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077143.217153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077143.729156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077143.729156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077143.729162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077143.729179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077144.049116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077144.049124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077144.049128:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077144.049136:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2849 00000100:00080000:1.0:1769077144.049146:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077144.049152:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077144.049186:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913415040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077144.049209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077144.049211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077144.049213:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077144.049216:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2849 00000100:00080000:1.0:1769077144.049218:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077144.049221:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077144.049235:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077144.049244:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913415168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077144.049251:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077144.049254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077144.049255:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077144.049258:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2849 00000100:00080000:1.0:1769077144.049260:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077144.049265:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049268:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913415040:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077144.049270:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077144.049279:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913415296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077144.049291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077144.049294:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077144.049295:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077144.049297:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2849 00000100:00080000:1.0:1769077144.049305:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077144.049307:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077144.049310:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077144.049319:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913415424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077144.049326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077144.049329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077144.049330:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077144.049332:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2849 00000100:00080000:1.0:1769077144.049335:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077144.049337:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077144.049341:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077144.049353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913415552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077144.049353:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077144.049362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077144.049365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077144.049371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077144.049374:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2849 00000100:00080000:1.0:1769077144.049377:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077144.049380:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077144.049383:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077144.049393:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913415680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077144.049398:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077144/real 1769077144] req@ffff9350100ca680 x1855008913415040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077144.049404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077144.049411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077144.049414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077144.049419:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2849 00000100:00080000:1.0:1769077144.049422:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077144.049425:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049428:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913415040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077144.049430:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077144.049442:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769077144.049446:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913415808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077144.049447:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077144.049463:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913415040:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077144.049471:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913415168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077144.049478:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000800:00000100:2.0:1769077144.049491:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049504:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077144/real 1769077144] req@ffff9350100c9f80 x1855008913415168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077144.049514:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913415168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077144.049520:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077144.049522:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077144.049525:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913415168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077144.049543:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913415296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077144.049554:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049561:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077144/real 1769077144] req@ffff9350100c9c00 x1855008913415296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077144.049569:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913415296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077144.049574:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077144.049576:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077144.049578:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913415296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077144.049581:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913415424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077144.049590:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049596:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077144/real 1769077144] req@ffff9350100cb100 x1855008913415424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077144.049603:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913415424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077144.049609:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077144.049611:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077144.049613:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913415424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077144.049616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913415552:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077144.049626:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049631:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077144/real 1769077144] req@ffff9350100c9180 x1855008913415552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077144.049638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913415552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077144.049644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077144.049646:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077144.049648:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913415552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077144.049651:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913415680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077144.049660:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049665:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077144/real 1769077144] req@ffff9350100c8a80 x1855008913415680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077144.049679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913415680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077144.049685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077144.049687:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077144.049706:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913415680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077144.049713:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913415808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077144.049731:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077144.049739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077144/real 1769077144] req@ffff9350100c8000 x1855008913415808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077144.049747:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913415808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077144.049753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077144.049755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077144.049757:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913415808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077144.241180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077144.753136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077144.753136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077144.753140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077144.753172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077145.073160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077145.265133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077145.777148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077145.777148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077145.777152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077145.777183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077146.097125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077146.289200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077146.801123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077146.801124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077146.801163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077146.801203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077147.121227:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077147.313172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077147.825163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077147.825201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077147.825206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077147.825234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077148.145116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077148.337156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077148.849119:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077148.849126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077148.849127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077148.849170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077149.169166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077149.169215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077149.169223:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077149.169227:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077149.169252:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2854 00000100:00080000:3.0:1769077149.169263:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077149.169269:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077149.169307:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913415936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077149.169333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077149.169337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077149.169338:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077149.169342:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2854 00000100:00080000:3.0:1769077149.169345:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077149.169347:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077149.169354:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077149.169363:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913416064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077149.169371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077149.169374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077149.169376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077149.169379:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2854 00000100:00080000:3.0:1769077149.169381:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077149.169383:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077149.169387:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077149.169394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913416192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077149.169405:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913415936:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077149.169405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077149.169411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077149.169414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077149.169421:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2854 00000100:00080000:3.0:1769077149.169423:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077149.169426:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077149.169429:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077149.169439:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913416320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077149.169445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077149.169448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077149.169450:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077149.169453:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2854 00000100:00080000:3.0:1769077149.169455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077149.169458:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077149.169461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077149.169473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913416448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077149.169483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077149.169503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077149.169505:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077149.169513:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2854 00000800:00000100:2.0:1769077149.169515:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077149.169520:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077149.169527:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077149.169534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077149.169551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913416576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077149.169565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077149.169571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077149.169572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077149/real 1769077149] req@ffff935002a7fb80 x1855008913415936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077149.169573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077149.169582:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2854 00000100:00080000:3.0:1769077149.169588:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077149.169596:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077149.169625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077149.169652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913415936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077149.169663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913416704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077149.169697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077149.169725:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077149.169733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913415936:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077149.169745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416064:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077149.169778:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077149.169796:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077149/real 1769077149] req@ffff935002a7df80 x1855008913416064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077149.169815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913416064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077149.169827:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077149.169832:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077149.169837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077149.169871:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077149.169899:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077149.169914:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077149/real 1769077149] req@ffff935002a7d180 x1855008913416192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077149.169930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913416192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077149.169958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077149.169962:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077149.169966:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077149.169973:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077149.169997:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077149.170027:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077149/real 1769077149] req@ffff935002a7dc00 x1855008913416320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077149.170044:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913416320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077149.170057:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077149.170061:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077149.170066:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077149.170074:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913416448:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077149.170101:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077149.170149:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077149/real 1769077149] req@ffff935002a7c000 x1855008913416448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077149.170168:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913416448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077149.170180:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077149.170184:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077149.170189:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913416448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077149.170197:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913416576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077149.170224:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077149.170254:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077149/real 1769077149] req@ffff935002a7f800 x1855008913416576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077149.170269:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913416576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077149.170280:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077149.170285:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077149.170289:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913416576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077149.170296:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913416704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077149.170320:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077149.170333:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077149/real 1769077149] req@ffff935002a7e300 x1855008913416704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077149.170347:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913416704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077149.170358:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077149.170361:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077149.170365:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913416704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077149.361124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077149.873156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077149.873156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077149.873161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077149.873197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077150.193138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077150.385182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077150.897121:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077150.897147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077150.897158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077150.897175:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077151.217168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077151.409122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077151.921167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077151.921167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077151.921180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077151.921220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077152.241156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077152.433191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077152.945160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077152.945160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077152.945183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077152.945230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077153.265151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077153.457120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077153.969187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077153.969275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077153.969281:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077153.969290:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077154.289151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077154.289214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077154.289221:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077154.289225:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077154.289233:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2859 00000100:00080000:0.0:1769077154.289244:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077154.289249:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077154.289285:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913416832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077154.289325:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077154.289328:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077154.289329:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077154.289332:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2859 00000100:00080000:0.0:1769077154.289334:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077154.289338:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077154.289346:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077154.289355:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913416960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077154.289364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077154.289382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077154.289383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077154.289389:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913416832:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077154.289390:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2859 00000100:00080000:0.0:1769077154.289396:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077154.289401:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077154.289405:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077154.289417:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913417088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077154.289424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077154.289428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077154.289446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077154.289451:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2859 00000100:00080000:0.0:1769077154.289455:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077154.289460:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077154.289465:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077154.289480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913417216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077154.289486:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077154.289511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077154.289534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077154.289537:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077154.289544:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2859 00000100:00080000:0.0:1769077154.289549:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077154.289554:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077154/real 1769077154] req@ffff9350103f9c00 x1855008913416832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077154.289575:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077154.289587:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077154.289605:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913416832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077154.289609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913417344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077154.289618:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077154.289627:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077154.289629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077154.289634:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913416832:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077154.289635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077154.289639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077154.289644:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2859 00000100:00080000:0.0:1769077154.289647:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077154.289651:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077154.289657:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077154.289660:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077154.289678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913417472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077154.289687:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077154.289698:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077154.289702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077154/real 1769077154] req@ffff9350103f9500 x1855008913416960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077154.289706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077154.289712:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077154.289721:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2859 00000100:00100000:2.0:1769077154.289722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913416960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077154.289730:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077154.289733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077154.289738:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077154.289739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077154.289746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913416960:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769077154.289750:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077154.289754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077154.289767:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913417600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077154.289774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077154.289789:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077154/real 1769077154] req@ffff9350103fbb80 x1855008913417088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077154.289798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913417088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077154.289808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077154.289811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077154.289813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077154.289825:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077154.289837:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077154.289843:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077154/real 1769077154] req@ffff9350103fad80 x1855008913417216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077154.289850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913417216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077154.289856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077154.289859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077154.289861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077154.289864:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913417344:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077154.289876:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077154.289885:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077154/real 1769077154] req@ffff9350103fb800 x1855008913417344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077154.289897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913417344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077154.289906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077154.289909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077154.289914:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913417344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077154.289934:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913417472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077154.289951:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077154.289959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077154/real 1769077154] req@ffff9350103f9f80 x1855008913417472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077154.289967:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913417472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077154.289973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077154.289975:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077154.289977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913417472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077154.289980:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913417600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077154.289990:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077154.289995:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077154/real 1769077154] req@ffff9350103f9c00 x1855008913417600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077154.290003:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913417600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077154.290008:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077154.290010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077154.290012:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913417600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077154.481164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077154.993161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077154.993189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077154.993192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077154.993193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077155.313174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077155.505144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077156.017141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077156.017158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077156.017164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077156.017227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077156.337172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077156.529157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077157.041146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077157.041166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077157.041196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077157.041293:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077157.361156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077157.553121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077158.065131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077158.065141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077158.065195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077158.065223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077158.385206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077158.577182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077159.089207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077159.089207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077159.089218:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077159.089299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077159.409136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077159.409136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077159.409146:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077159.409150:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077159.409159:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2865 00000100:00080000:3.0:1769077159.409169:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077159.409175:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077159.409210:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913417728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077159.409234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077159.409237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077159.409238:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077159.409241:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2865 00000100:00080000:3.0:1769077159.409243:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077159.409246:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077159.409254:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077159.409263:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913417856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077159.409271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077159.409273:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913417728:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077159.409274:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077159.409278:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077159.409286:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2865 00000100:00080000:3.0:1769077159.409288:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077159.409290:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077159.409294:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077159.409303:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913417984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077159.409310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077159.409313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077159.409315:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077159.409335:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2865 00000100:00080000:3.0:1769077159.409349:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077159.409353:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077159.409359:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077159.409362:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077159.409380:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913418112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077159.409393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077159.409398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1769077159.409399:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077159/real 1769077159] req@ffff935002a7ca80 x1855008913417728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077159.409402:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077159.409411:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2865 00000100:00080000:3.0:1769077159.409417:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077159.409421:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077159.409425:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913417728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077159.409428:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077159.409455:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077159.409474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913418240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077159.409478:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077159.409486:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913417728:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077159.409489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077159.409493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077159.409495:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077159.409498:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2865 00000100:00080000:3.0:1769077159.409502:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077159.409505:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077159.409509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077159.409511:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077159.409520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913418368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077159.409535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077159.409538:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077159.409543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077159.409546:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077159.409552:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2865 00000100:00100000:2.0:1769077159.409554:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077159/real 1769077159] req@ffff935002a7c380 x1855008913417856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077159.409560:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077159.409568:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913417856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077159.409569:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077159.409579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077159.409580:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077159.409604:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077159.409608:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077159.409616:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913418496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077159.409618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077159.409644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077159.409653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077159/real 1769077159] req@ffff935002a7f100 x1855008913417984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077159.409661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913417984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077159.409667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077159.409669:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077159.409671:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913417984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077159.409675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913418112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077159.409686:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077159.409692:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077159/real 1769077159] req@ffff935002a7c700 x1855008913418112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077159.409700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913418112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077159.409705:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077159.409707:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077159.409709:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913418112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077159.409713:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913418240:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077159.409723:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077159.409729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077159/real 1769077159] req@ffff935002a7e300 x1855008913418240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077159.409737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913418240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077159.409743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077159.409744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077159.409746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913418240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077159.409776:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913418368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077159.409788:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077159.409794:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077159/real 1769077159] req@ffff935002a7f800 x1855008913418368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077159.409802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913418368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077159.409807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077159.409809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077159.409811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913418368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077159.409814:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913418496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077159.409823:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077159.409829:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077159/real 1769077159] req@ffff935002a7c000 x1855008913418496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077159.409836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913418496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077159.409846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077159.409848:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077159.409850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913418496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077159.601125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077160.113120:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077160.113136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077160.113183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077160.113191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077160.433178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077160.625160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077161.137179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077161.137179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077161.137179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077161.137215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077161.457157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077161.649141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077162.161152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077162.161152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077162.161153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077162.161185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077162.481182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077162.673215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077163.185124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077163.185127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077163.185140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077163.185179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077163.505122:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077163.697204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077164.209216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077164.209218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077164.209220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077164.209316:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077164.529199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077164.529247:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077164.529260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077164.529266:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077164.529276:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2870 00000100:00080000:0.0:1769077164.529291:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077164.529299:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077164.529386:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913418624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077164.529435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077164.529443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077164.529447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077164.529455:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2870 00000100:00080000:0.0:1769077164.529464:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077164.529474:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077164.529491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077164.529507:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913418624:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077164.529524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913418752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077164.529542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077164.529548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077164.529553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077164.529559:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2870 00000100:00080000:0.0:1769077164.529574:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077164.529582:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077164.529592:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077164.529606:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077164.529621:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913418880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077164.529643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077164.529651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077164.529656:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077164.529663:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077164/real 1769077164] req@ffff9350103f9180 x1855008913418624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077164.529671:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2870 00000100:00080000:0.0:1769077164.529697:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077164.529707:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077164.529716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913418624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077164.529720:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1769077164.529741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077164.529750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913419008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077164.529751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077164.529764:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913418624:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077164.529786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077164.529800:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077164.529803:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077164.529811:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2870 00000100:00080000:0.0:1769077164.529817:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077164.529824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913418752:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077164.529836:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077164.529845:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077164.529876:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913419136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077164.529890:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077164.529899:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769077164.529901:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077164.529903:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077164.529916:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2870 00000100:00080000:0.0:1769077164.529923:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077164.529929:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077164.529936:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077164.529948:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077164/real 1769077164] req@ffff9350103fb100 x1855008913418752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077164.529955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913419264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077164.530002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077164.530007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077164.530013:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077164.530013:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913418752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077164.530039:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2870 00000100:00080000:0.0:1769077164.530089:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077164.530098:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077164.530101:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769077164.530105:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077164.530118:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077164.530131:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913419392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077164.530150:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913418752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077164.530169:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913418880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077164.530217:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077164.530239:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077164/real 1769077164] req@ffff9350103f8700 x1855008913418880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077164.530256:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913418880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077164.530267:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077164.530271:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077164.530275:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913418880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077164.530282:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077164.530320:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077164.530333:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077164/real 1769077164] req@ffff9350103f8380 x1855008913419008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077164.530379:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913419008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077164.530391:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077164.530395:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077164.530399:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077164.530424:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913419136:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077164.530451:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077164.530464:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077164/real 1769077164] req@ffff9350103fb480 x1855008913419136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077164.530479:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913419136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077164.530489:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077164.530492:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077164.530496:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913419136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077164.530502:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913419264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077164.530519:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077164.530529:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077164/real 1769077164] req@ffff9350103faa00 x1855008913419264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077164.530542:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913419264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077164.530551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077164.530554:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077164.530566:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913419264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077164.530572:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913419392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077164.530588:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077164.530599:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077164/real 1769077164] req@ffff9350103f9180 x1855008913419392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077164.530613:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913419392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077164.530622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077164.530625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077164.530628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913419392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077164.721159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077165.233148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077165.233149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077165.233149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077165.233195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077165.553170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077165.745153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077166.257126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077166.257126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077166.257149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077166.257167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077166.577152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077166.769141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077167.281184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077167.281184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077167.281195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077167.281263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077167.601176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077167.793166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077168.305144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077168.305155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077168.305156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077168.305182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077168.625206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077168.817168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077169.329141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077169.329143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077169.329144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077169.329175:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077169.649120:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077169.649142:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077169.649149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077169.649153:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077169.649161:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2875 00000100:00080000:3.0:1769077169.649171:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077169.649176:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077169.649210:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913419520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077169.649233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077169.649236:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077169.649238:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077169.649245:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2875 00000100:00080000:3.0:1769077169.649248:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077169.649251:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077169.649275:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077169.649289:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913419520:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077169.649289:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913419648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077169.649304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077169.649306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077169.649308:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077169.649312:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2875 00000100:00080000:3.0:1769077169.649315:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077169.649317:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077169.649322:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077169.649330:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913419776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077169.649337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077169.649340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077169.649341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077169.649345:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2875 00000100:00080000:3.0:1769077169.649348:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077169.649350:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077169.649354:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077169.649355:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077169.649365:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913419904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077169.649372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077169.649375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077169.649376:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077169.649380:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2875 00000100:00080000:3.0:1769077169.649382:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077169.649384:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077169.649387:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077169.649393:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077169/real 1769077169] req@ffff935002a7dc00 x1855008913419520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077169.649398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913420032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077169.649416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077169.649419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077169.649420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077169.649423:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913419520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077169.649425:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2875 00000100:00080000:3.0:1769077169.649430:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077169.649436:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077169.649442:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077169.649443:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077169.649451:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077169.649454:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913420160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077169.649457:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913419520:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077169.649469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077169.649472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077169.649474:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077169.649477:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2875 00000100:00080000:3.0:1769077169.649479:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077169.649482:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077169.649485:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419648:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077169.649487:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077169.649499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913420288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077169.649510:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077169.649522:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077169/real 1769077169] req@ffff935002a7d180 x1855008913419648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077169.649537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913419648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077169.649551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077169.649554:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077169.649557:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077169.649568:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077169.649589:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077169.649601:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077169/real 1769077169] req@ffff935002a7df80 x1855008913419776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077169.649615:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913419776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077169.649624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077169.649627:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077169.649629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077169.649634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077169.649651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077169.649662:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077169/real 1769077169] req@ffff935002a7fb80 x1855008913419904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077169.649690:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913419904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077169.649708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077169.649712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077169.649716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913419904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077169.649723:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077169.649740:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077169.649749:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077169/real 1769077169] req@ffff935002a7c000 x1855008913420032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077169.649757:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913420032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077169.649763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077169.649766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077169.649768:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077169.649772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077169.649782:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077169.649788:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077169/real 1769077169] req@ffff935002a7f800 x1855008913420160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077169.649796:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913420160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077169.649801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077169.649803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077169.649806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077169.649831:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077169.649841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077169.649848:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077169/real 1769077169] req@ffff935002a7e300 x1855008913420288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077169.649859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913420288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077169.649868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077169.649871:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077169.649875:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077169.841215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077170.353120:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077170.353150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077170.353289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077170.353290:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077170.673127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077170.865150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077171.377203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077171.377266:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077171.377287:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077171.377301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077171.697155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077171.889259:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077172.401169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077172.401169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077172.401200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077172.401320:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077172.721195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077172.913196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077173.425139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077173.425140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077173.425223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077173.425246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077173.745175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077173.937268:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077174.449255:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077174.449265:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077174.449273:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077174.449300:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077174.769235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077174.769247:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077174.769254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077174.769264:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077174.769279:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2880 00000100:00080000:2.0:1769077174.769295:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077174.769304:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077174.769352:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913420416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077174.769387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077174.769395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077174.769397:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077174.769402:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2880 00000100:00080000:2.0:1769077174.769406:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077174.769411:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077174.769425:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077174.769451:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913420544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077174.769454:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913420416:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077174.769484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077174.769491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077174.769495:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077174.769505:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2880 00000100:00080000:2.0:1769077174.769513:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077174.769520:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077174.769531:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077174.769550:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913420672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077174.769563:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077174.769571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077174.769578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077174.769581:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077174.769587:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2880 00000100:00080000:2.0:1769077174.769594:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077174.769599:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077174.769605:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077174.769638:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913420800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077174.769657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077174.769664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769077174.769672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077174/real 1769077174] req@ffff935002a62a00 x1855008913420416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077174.769673:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077174.769693:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2880 00000100:00080000:2.0:1769077174.769708:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077174.769718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913420416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077174.769722:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077174.769744:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077174.769748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077174.769761:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077174.769770:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913420416:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077174.769780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913420928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077174.769813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077174.769818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077174.769820:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077174.769826:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2880 00000100:00080000:2.0:1769077174.769832:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077174.769836:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913420544:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077174.769841:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077174.769867:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077174.769887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913421056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077174.769901:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077174.769921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077174.769928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077174.769933:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077174.769942:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2880 00000100:00080000:2.0:1769077174.769949:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077174.769956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077174/real 1769077174] req@ffff935002a60000 x1855008913420544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077174.769965:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077174.769999:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077174.770014:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913420544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077174.770027:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913421184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077174.770039:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077174.770056:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077174.770066:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913420544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077174.770078:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913420672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077174.770198:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077174.770223:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077174/real 1769077174] req@ffff935002a62300 x1855008913420672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077174.770246:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913420672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077174.770260:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077174.770266:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077174.770272:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913420672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077174.770281:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913420800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077174.770328:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077174.770344:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077174/real 1769077174] req@ffff935002a61500 x1855008913420800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077174.770359:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913420800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077174.770370:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077174.770374:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077174.770378:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913420800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077174.770385:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420928:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077174.770404:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077174.770414:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077174/real 1769077174] req@ffff935002a61880 x1855008913420928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077174.770428:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913420928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077174.770438:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077174.770442:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077174.770446:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913420928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077174.770474:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077174.770522:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077174.770540:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077174/real 1769077174] req@ffff935002a61180 x1855008913421056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077174.770562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913421056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077174.770577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077174.770582:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077174.770590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077174.770600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077174.770649:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077174.770664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077174/real 1769077174] req@ffff935002a63100 x1855008913421184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077174.770680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913421184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077174.770690:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077174.770693:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077174.770697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077174.961124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077175.473143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077175.473143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077175.473143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077175.473199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077175.793248:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077175.985244:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077176.497207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077176.497207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077176.497207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077176.497285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077176.817205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077177.009156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077177.521196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077177.521202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077177.521210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077177.521239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077177.841198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077178.033288:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077178.545261:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077178.545261:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077178.545261:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077178.545328:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077178.865149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077179.057156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077179.569178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077179.569180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077179.569191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077179.569230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077179.889166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077179.889173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077179.889176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077179.889182:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077179.889192:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2885 00000100:00080000:3.0:1769077179.889202:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077179.889209:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077179.889257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913421312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077179.889293:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077179.889296:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077179.889298:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077179.889302:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2885 00000100:00080000:3.0:1769077179.889306:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077179.889309:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077179.889318:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077179.889337:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913421440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077179.889347:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913421312:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077179.889348:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077179.889357:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077179.889359:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077179.889363:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2885 00000100:00080000:3.0:1769077179.889365:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077179.889368:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077179.889372:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077179.889381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913421568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077179.889389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077179.889392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077179.889393:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077179.889395:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2885 00000100:00080000:3.0:1769077179.889399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077179.889401:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077179.889406:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077179.889418:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913421696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077179.889419:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077179.889429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077179.889432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077179.889435:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077179.889438:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2885 00000100:00080000:3.0:1769077179.889440:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077179.889442:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077179.889445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077179.889455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913421824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077179.889462:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077179.889464:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff935002a7c700 x1855008913421312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077179.889466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077179.889469:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077179.889477:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2885 00000100:00080000:3.0:1769077179.889483:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077179.889489:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077179.889492:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077179.889503:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913421312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077179.889504:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913421952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077179.889538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077179.889541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077179.889545:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077179.889546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077179.889551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077179.889554:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913421312:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077179.889558:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2885 00000100:00080000:3.0:1769077179.889566:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077179.889569:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077179.889572:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077179.889580:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913422080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077179.889590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913421440:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077179.889614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077179.889625:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff935002a7f100 x1855008913421440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077179.889635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913421440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077179.889642:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077179.889644:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077179.889647:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913421440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077179.889652:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913421568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077179.889664:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077179.889670:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff935002a7c380 x1855008913421568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077179.889681:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913421568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077179.889687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077179.889690:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077179.889693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913421568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077179.889697:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913421696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077179.889710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077179.889717:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff935002a7ca80 x1855008913421696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077179.889724:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913421696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077179.889730:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077179.889732:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077179.889734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913421696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077179.889737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077179.889748:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077179.889754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff935002a7e300 x1855008913421824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077179.889761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913421824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077179.889767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077179.889769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077179.889771:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077179.889774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077179.889784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077179.889790:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff935002a7f800 x1855008913421952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077179.889802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913421952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077179.889808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077179.889809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077179.889811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913421952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077179.889827:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077179.889836:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077179.889842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff935002a7c000 x1855008913422080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077179.889849:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913422080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077179.889857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077179.889860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077179.889862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077180.081164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077180.593174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077180.593174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077180.593174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077180.593200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077180.913187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077181.105153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077181.617182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077181.617184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077181.617207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077181.617230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077181.937236:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077182.129214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077182.641178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077182.641185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077182.641284:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077182.641322:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077182.961125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077183.153161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077183.665140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077183.665140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077183.665140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077183.665242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077183.985136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077184.177154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077184.689106:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077184.689149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077184.689157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077184.689180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077185.009131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077185.009131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077185.009141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077185.009146:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077185.009154:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2890 00000100:00080000:0.0:1769077185.009164:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077185.009171:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077185.009207:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913422208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077185.009233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077185.009235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077185.009237:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077185.009240:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2890 00000100:00080000:0.0:1769077185.009242:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077185.009244:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077185.009251:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077185.009261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913422336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077185.009270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077185.009272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077185.009274:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077185.009277:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913422208:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077185.009279:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2890 00000100:00080000:0.0:1769077185.009283:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077185.009288:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077185.009291:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077185.009300:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913422464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077185.009305:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077185.009308:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077185.009310:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077185.009312:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2890 00000100:00080000:0.0:1769077185.009314:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077185.009317:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077185.009320:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077185.009327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913422592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077185.009332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077185.009335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077185.009336:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077185.009339:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2890 00000100:00080000:0.0:1769077185.009344:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077185.009348:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077185.009354:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077185.009367:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913422720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077185.009378:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077185.009383:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913422336:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077185.009384:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077185.009389:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077185.009395:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2890 00000100:00080000:0.0:1769077185.009399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077185.009402:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077185.009408:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077185.009415:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913422464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077185.009422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913422848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077185.009431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077185.009435:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077185.009436:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913422592:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077185.009439:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077185.009444:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2890 00000100:00080000:0.0:1769077185.009447:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077185.009450:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077185.009455:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077185.009459:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077185.009467:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913422976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077185.009485:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077185.009503:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077185.201168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077185.713131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077185.713131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077185.713161:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077185.713169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077186.033144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077186.225135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077186.737146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077186.737148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077186.737148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077186.737190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077187.057128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077187.249152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077187.761122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077187.761124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077187.761149:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077187.761169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077188.081126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077188.081302:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077188.081339:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077188.081359:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077188.081413:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077188.081423:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077188.081433:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077188.081442:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077188.081451:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077188.081459:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077188.081739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077185/real 1769077188] req@ffff9350103f9500 x1855008913422848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077188.081770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913422848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077188.081783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077188.081789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077188.081793:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077188.081802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077185/real 1769077188] req@ffff9350103f9180 x1855008913422976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077188.081815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913422976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077188.081824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077188.081828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077188.081832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077188.081846:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077185/real 1769077188] req@ffff9350103f9f80 x1855008913422336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077188.081859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913422336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077188.081868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077188.081873:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077188.081876:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913422336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077188.081885:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077185/real 1769077188] req@ffff9350103fb800 x1855008913422464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077188.081898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913422464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077188.081906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077188.081910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077188.081913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913422464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077188.081920:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077185/real 1769077188] req@ffff9350103fad80 x1855008913422592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077188.081932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913422592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077188.081941:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077188.081945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077188.081962:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913422592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077188.081970:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077185/real 1769077188] req@ffff9350103fbb80 x1855008913422720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077188.081983:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913422720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077188.081991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077188.081995:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077188.081999:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913422720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077188.082006:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077185/real 1769077188] req@ffff9350103f9c00 x1855008913422208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077188.082019:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913422208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077188.082027:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077188.082030:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077188.082034:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913422208:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077188.273165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077188.785162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077188.785162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077188.785187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077188.785193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077189.105185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077189.297175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077189.809193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077189.809194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077189.809195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077189.809224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077190.129199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077190.129234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077190.129246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077190.129252:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077190.129264:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2895 00000100:00080000:3.0:1769077190.129277:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077190.129285:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077190.129323:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913423104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077190.129351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077190.129355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077190.129357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077190.129362:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2895 00000100:00080000:3.0:1769077190.129366:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077190.129371:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077190.129385:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913423104:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077190.129388:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077190.129405:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913423232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077190.129417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077190.129421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077190.129423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077190.129428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2895 00000100:00080000:3.0:1769077190.129448:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077190.129453:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077190.129459:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077190.129464:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077190.129476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913423360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077190.129488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077190.129492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077190.129494:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077190.129499:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2895 00000100:00080000:3.0:1769077190.129503:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077190.129508:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077190.129514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077190.129517:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077190/real 1769077190] req@ffff935002a7fb80 x1855008913423104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077190.129529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913423488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077190.129541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077190.129545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913423104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077190.129546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077190.129551:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077190.129562:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2895 00000100:00080000:2.0:1769077190.129567:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077190.129567:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077190.129576:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077190.129579:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769077190.129583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077190.129589:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913423104:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077190.129601:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913423616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077190.129617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077190.129622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077190.129626:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913423232:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077190.129627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077190.129637:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2895 00000100:00080000:3.0:1769077190.129643:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077190.129647:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077190.129653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077190.129662:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077190.129670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913423744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077190.129682:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077190.129683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077190/real 1769077190] req@ffff935002a7df80 x1855008913423232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077190.129692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077190.129697:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077190.129707:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2895 00000100:00080000:3.0:1769077190.129715:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077190.129722:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077190.129726:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913423232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077190.129730:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077190.129744:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077190.129749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913423872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077190.129753:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077190.129761:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913423232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077190.129775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913423360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077190.129805:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077190.129822:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077190/real 1769077190] req@ffff935002a7d180 x1855008913423360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077190.129835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913423360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077190.129843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077190.129847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077190.129851:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913423360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077190.129857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913423488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077190.129877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077190.129889:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077190/real 1769077190] req@ffff935002a7dc00 x1855008913423488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077190.129901:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913423488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077190.129931:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077190.129935:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077190.129938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913423488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077190.129944:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913423616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077190.129966:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077190.129978:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077190/real 1769077190] req@ffff935002a7c000 x1855008913423616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077190.129990:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913423616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077190.129999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077190.130003:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077190.130006:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913423616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077190.130026:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913423744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077190.130083:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077190.130101:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077190/real 1769077190] req@ffff935002a7f800 x1855008913423744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077190.130114:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913423744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077190.130123:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077190.130127:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077190.130134:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913423744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077190.130143:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913423872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077190.130166:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077190.130180:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077190/real 1769077190] req@ffff935002a7e300 x1855008913423872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077190.130192:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913423872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077190.130201:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077190.130204:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077190.130207:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913423872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077190.321175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077190.833149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077190.833149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077190.833152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077190.833196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077191.153168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077191.345149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077191.857149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077191.857150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077191.857150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077191.857186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077192.177177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077192.369149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077192.881143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077192.881143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077192.881143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077192.881196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077193.201174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077193.393149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077193.905153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077193.905155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077193.905157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077193.905179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077194.225166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077194.417118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077194.929145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077194.929145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077194.929145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077194.929172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077195.249145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077195.249148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077195.249156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077195.249160:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077195.249169:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2900 00000100:00080000:3.0:1769077195.249179:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077195.249186:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077195.249219:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913424000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077195.249243:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077195.249246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077195.249247:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077195.249251:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2900 00000100:00080000:3.0:1769077195.249253:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077195.249255:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077195.249263:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077195.249271:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913424128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077195.249278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077195.249281:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077195.249282:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077195.249286:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2900 00000100:00080000:3.0:1769077195.249287:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077195.249290:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077195.249293:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077195.249301:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913424256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077195.249309:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913424000:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077195.249311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077195.249317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077195.249320:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077195.249323:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2900 00000100:00080000:3.0:1769077195.249325:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077195.249327:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077195.249330:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077195.249339:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913424384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077195.249346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077195.249348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077195.249350:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077195.249352:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2900 00000100:00080000:3.0:1769077195.249354:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077195.249356:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077195.249359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077195.249366:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913424512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077195.249372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077195.249375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077195.249377:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077195.249380:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2900 00000100:00080000:3.0:1769077195.249383:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077195.249385:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077195.249389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077195.249393:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077195.249400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913424640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077195.249407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077195.249409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077195.249411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077195.249414:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2900 00000100:00080000:3.0:1769077195.249419:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077195.249421:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077195.249424:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077195.249433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913424768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077195.249445:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077195/real 1769077195] req@ffff935002a7ca80 x1855008913424000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077195.249467:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913424000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077195.249479:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077195.249484:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077195.249489:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913424000:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077195.249497:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913424128:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077195.249523:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077195.249537:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077195/real 1769077195] req@ffff935002a7c380 x1855008913424128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077195.249552:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913424128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077195.249563:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077195.249567:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077195.249571:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913424128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077195.249596:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913424256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077195.249627:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077195.249639:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077195/real 1769077195] req@ffff935002a7f100 x1855008913424256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077195.249652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913424256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077195.249661:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077195.249665:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077195.249668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913424256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077195.249675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913424384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077195.249692:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077195.249704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077195/real 1769077195] req@ffff935002a7c700 x1855008913424384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077195.249717:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913424384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077195.249726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077195.249730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077195.249733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913424384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077195.249740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913424512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077195.249759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077195.249770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077195/real 1769077195] req@ffff935002a7e300 x1855008913424512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077195.249785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913424512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077195.249804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077195.249808:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077195.249811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913424512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077195.249818:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913424640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077195.249873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077195.249885:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077195/real 1769077195] req@ffff935002a7f800 x1855008913424640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077195.249899:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913424640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077195.249908:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077195.249920:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077195.249929:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913424640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077195.249936:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913424768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077195.249957:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077195.249969:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077195/real 1769077195] req@ffff935002a7c000 x1855008913424768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077195.249981:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913424768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077195.249990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077195.249993:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077195.249997:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913424768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077195.441143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077195.953149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077195.953150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077195.953150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077195.953191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077196.273141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077196.465124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077196.977222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077196.977222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077196.977289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077196.977305:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077197.297152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077197.489143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077198.001213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077198.001213:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077198.001231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077198.001257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077198.321149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077198.513177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077199.025142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077199.025154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077199.025167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077199.025219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077199.345169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077199.537130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077200.049107:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077200.049110:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077200.049138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077200.049198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077200.369124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077200.369177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077200.369184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077200.369187:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077200.369195:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2906 00000100:00080000:2.0:1769077200.369205:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077200.369211:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077200.369246:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913424896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077200.369272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077200.369275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077200.369277:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077200.369280:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2906 00000100:00080000:2.0:1769077200.369283:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077200.369286:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077200.369293:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077200.369302:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913425024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077200.369311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077200.369313:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913424896:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077200.369315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077200.369319:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077200.369327:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2906 00000100:00080000:2.0:1769077200.369330:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077200.369332:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077200.369335:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077200.369345:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913425152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077200.369352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077200.369354:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077200.369355:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077200.369359:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2906 00000100:00080000:2.0:1769077200.369361:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077200.369364:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077200.369367:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077200.369374:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913425280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077200.369380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077200.369382:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077200.369384:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077200.369387:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2906 00000100:00080000:2.0:1769077200.369390:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077200.369394:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077200.369406:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077200.369411:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077200.369424:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913425408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077200.369433:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077200/real 1769077200] req@ffff935002a60a80 x1855008913424896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077200.369437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077200.369449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077200.369452:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077200.369457:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913424896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077200.369461:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2906 00000100:00080000:0.0:1769077200.369470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077200.369471:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077200.369475:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077200.369481:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077200.369482:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913424896:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769077200.369494:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077200.369505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077200.369510:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913425536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077200.369522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077200.369525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769077200.369528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077200.369529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077200.369535:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2906 00000100:00080000:2.0:1769077200.369541:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077200.369544:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077200/real 1769077200] req@ffff935002a62680 x1855008913425024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077200.369547:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077200.369555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077200.369561:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913425024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077200.369568:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913425664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077200.369570:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077200.369578:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077200.369585:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077200.369591:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077200.369605:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077200.369611:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077200/real 1769077200] req@ffff935002a61f80 x1855008913425152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077200.369624:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913425152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077200.369629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077200.369631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077200.369633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077200.369636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077200.369645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077200.369651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077200/real 1769077200] req@ffff935002a61c00 x1855008913425280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077200.369658:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913425280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077200.369663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077200.369665:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077200.369667:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077200.369670:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913425408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077200.369679:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077200.369684:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077200/real 1769077200] req@ffff935002a63100 x1855008913425408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077200.369692:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913425408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077200.369707:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077200.369711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077200.369714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913425408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077200.369731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913425536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077200.369766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077200.369779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077200/real 1769077200] req@ffff935002a61180 x1855008913425536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077200.369790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913425536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077200.369796:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077200.369798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077200.369800:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913425536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077200.369804:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913425664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077200.369818:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077200.369825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077200/real 1769077200] req@ffff935002a61880 x1855008913425664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077200.369833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913425664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077200.369838:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077200.369839:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077200.369842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913425664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077200.561161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077201.073124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077201.073129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077201.073132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077201.073176:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077201.393155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077201.585160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077202.097135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077202.097147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077202.097149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077202.097166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077202.417173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077202.609204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077203.121147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077203.121147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077203.121153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077203.121217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077203.441148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077203.633112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077204.145160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077204.145160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077204.145160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077204.145196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077204.465120:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077204.657122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077205.169109:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077205.169129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077205.169130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077205.169165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077205.489162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077205.489168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077205.489175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077205.489180:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077205.489189:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2911 00000100:00080000:3.0:1769077205.489201:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077205.489210:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077205.489263:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913425792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077205.489293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077205.489296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077205.489298:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077205.489301:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2911 00000100:00080000:3.0:1769077205.489305:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077205.489309:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077205.489317:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077205.489330:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913425920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077205.489339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077205.489340:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913425792:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077205.489342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077205.489344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077205.489351:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2911 00000100:00080000:3.0:1769077205.489354:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077205.489357:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077205.489361:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077205.489372:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913426048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077205.489380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077205.489383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077205.489384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077205.489386:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2911 00000100:00080000:3.0:1769077205.489389:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077205.489391:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077205.489394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077205.489401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913426176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077205.489408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077205.489413:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077205.489424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077205.489427:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077205.489434:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2911 00000100:00080000:3.0:1769077205.489439:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077205.489444:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077205.489451:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077205.489453:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077205/real 1769077205] req@ffff935002a7dc00 x1855008913425792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077205.489469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913426304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077205.489481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077205.489484:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913425792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077205.489488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077205.489492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077205.489502:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2911 00000100:00080000:0.0:1769077205.489504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077205.489507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077205.489513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077205.489516:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077205.489523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077205.489524:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913425792:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077205.489537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913426432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077205.489544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077205.489547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077205.489548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077205.489551:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2911 00000100:00080000:3.0:1769077205.489554:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077205.489557:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425920:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077205.489559:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077205.489567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077205.489577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913426560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077205.489592:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077205.489609:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077205/real 1769077205] req@ffff935002a7d180 x1855008913425920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077205.489623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913425920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077205.489633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077205.489636:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077205.489641:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913425920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077205.489648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077205.489688:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077205.489701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077205/real 1769077205] req@ffff935002a7df80 x1855008913426048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077205.489714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913426048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077205.489723:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077205.489727:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077205.489730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077205.489737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077205.489758:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077205.489771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077205/real 1769077205] req@ffff935002a7fb80 x1855008913426176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077205.489783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913426176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077205.489793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077205.489797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077205.489801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077205.489807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913426304:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077205.489825:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077205.489834:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077205/real 1769077205] req@ffff935002a7c000 x1855008913426304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077205.489842:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913426304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077205.489848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077205.489850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077205.489853:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913426304:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077205.489857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913426432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077205.489867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077205.489873:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077205/real 1769077205] req@ffff935002a7f800 x1855008913426432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077205.489882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913426432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077205.489887:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077205.489889:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077205.489891:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913426432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077205.489906:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913426560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077205.489921:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077205.489931:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077205/real 1769077205] req@ffff935002a7e300 x1855008913426560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077205.489943:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913426560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077205.489951:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077205.489955:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077205.489958:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913426560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077205.681156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077206.193149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077206.193149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077206.193149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077206.193186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077206.513172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077206.705124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077207.217118:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077207.217118:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077207.217134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077207.217150:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077207.537130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077207.729134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077208.241122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077208.241124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077208.241128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077208.241171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077208.561153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077208.753181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077209.265128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077209.265155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077209.265178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077209.265201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077209.585148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077209.777132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077210.289172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077210.289172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077210.289177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077210.289230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077210.609150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077210.609194:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077210.609200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077210.609204:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077210.609211:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2916 00000100:00080000:2.0:1769077210.609221:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077210.609227:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077210.609262:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913426688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077210.609287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077210.609290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077210.609291:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077210.609294:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2916 00000100:00080000:2.0:1769077210.609297:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077210.609299:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077210.609306:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077210.609314:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913426816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077210.609321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077210.609324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077210.609325:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077210.609328:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2916 00000100:00080000:2.0:1769077210.609330:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077210.609332:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077210.609337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077210.609343:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913426688:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077210.609346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913426944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077210.609355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077210.609358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077210.609359:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077210.609361:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2916 00000100:00080000:2.0:1769077210.609364:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077210.609366:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077210.609369:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077210.609377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913427072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077210.609383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077210.609386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077210.609387:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077210.609390:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2916 00000100:00080000:2.0:1769077210.609393:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077210.609395:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077210.609398:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077210.609407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913427200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077210.609413:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077210.609420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077210.609423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077210.609425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077210.609428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2916 00000100:00080000:2.0:1769077210.609431:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077210.609434:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077210.609437:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077210.609445:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913427328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077210.609456:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077210/real 1769077210] req@ffff935002a61500 x1855008913426688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077210.609458:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077210.609465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077210.609467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077210.609470:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2916 00000100:00080000:2.0:1769077210.609472:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077210.609473:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077210.609478:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077210.609487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913427456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077210.609493:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913426688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077210.609504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077210.609508:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077210.609512:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913426688:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077210.609518:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426816:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077210.609535:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077210.609543:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077210/real 1769077210] req@ffff935002a62300 x1855008913426816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077210.609551:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913426816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077210.609558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077210.609560:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077210.609563:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077210.609611:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077210.609627:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077210.609634:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077210/real 1769077210] req@ffff935002a60000 x1855008913426944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077210.609642:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913426944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077210.609648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077210.609651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077210.609653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913426944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077210.609656:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077210.609665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077210.609671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077210/real 1769077210] req@ffff935002a62a00 x1855008913427072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077210.609678:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913427072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077210.609683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077210.609685:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077210.609687:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077210.609692:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913427200:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077210.609701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077210.609707:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077210/real 1769077210] req@ffff935002a61880 x1855008913427200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077210.609714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913427200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077210.609724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077210.609727:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077210.609731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913427200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077210.609737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913427328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077210.609757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077210.609769:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077210/real 1769077210] req@ffff935002a61180 x1855008913427328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077210.609783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913427328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077210.609790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077210.609793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077210.609795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913427328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077210.609800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913427456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077210.609814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077210.609822:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077210/real 1769077210] req@ffff935002a63100 x1855008913427456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077210.609829:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913427456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077210.609835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077210.609837:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077210.609839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913427456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077210.801168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077211.313239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077211.313240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077211.313240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077211.313360:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077211.633177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077211.825138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077212.337132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077212.337133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077212.337181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077212.337183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077212.657128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077212.849139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077213.361151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077213.361152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077213.361185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077213.361230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077213.681158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077213.873118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077214.385168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077214.385169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077214.385221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077214.385277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077214.705156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077214.897164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077215.409161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077215.409161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077215.409180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077215.409203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077215.729134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077215.729151:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077215.729157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077215.729161:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077215.729171:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2921 00000100:00080000:0.0:1769077215.729182:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077215.729189:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077215.729223:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913427584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077215.729249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077215.729252:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077215.729254:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077215.729257:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2921 00000100:00080000:0.0:1769077215.729259:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077215.729263:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077215.729271:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077215.729281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913427712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077215.729288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077215.729291:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077215.729292:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077215.729294:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2921 00000100:00080000:0.0:1769077215.729296:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077215.729302:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077215.729304:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913427584:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769077215.729305:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077215.729316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913427840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077215.729324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077215.729326:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077215.729328:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077215.729330:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2921 00000100:00080000:0.0:1769077215.729332:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077215.729334:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077215.729337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077215.729346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913427968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077215.729353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077215.729356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077215.729357:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077215.729360:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2921 00000100:00080000:0.0:1769077215.729363:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077215.729365:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077215.729368:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077215.729377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913428096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077215.729382:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077215.729391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077215.729394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077215.729396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077215.729399:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2921 00000100:00080000:0.0:1769077215.729402:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077215.729405:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077215.729408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077215.729418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913428224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077215.729424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077215.729427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077215.729430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077215.729432:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077215/real 1769077215] req@ffff9350103faa00 x1855008913427584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077215.729434:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2921 00000100:00080000:0.0:1769077215.729439:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077215.729444:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077215.729452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077215.729465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913428352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077215.729472:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913427584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077215.729503:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077215.729509:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077215.729513:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913427584:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077215.729522:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427712:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077215.729550:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077215.729565:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077215/real 1769077215] req@ffff9350103fb480 x1855008913427712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077215.729578:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913427712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077215.729587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077215.729591:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077215.729595:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077215.729619:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077215.729641:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077215.729654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077215/real 1769077215] req@ffff9350103f8380 x1855008913427840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077215.729666:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913427840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077215.729675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077215.729678:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077215.729682:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077215.729688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077215.729707:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077215.729719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077215/real 1769077215] req@ffff9350103f8700 x1855008913427968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077215.729732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913427968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077215.729742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077215.729745:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077215.729748:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913427968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077215.729754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428096:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077215.729775:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077215.729786:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077215/real 1769077215] req@ffff9350103fb100 x1855008913428096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077215.729798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913428096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077215.729818:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077215.729822:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077215.729824:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077215.729829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077215.729846:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077215.729854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077215/real 1769077215] req@ffff9350103f9c00 x1855008913428224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077215.729862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913428224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077215.729867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077215.729869:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077215.729872:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077215.729876:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077215.729886:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077215.729891:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077215/real 1769077215] req@ffff9350103fbb80 x1855008913428352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077215.729900:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913428352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077215.729908:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077215.729912:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077215.729915:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077215.921133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077216.433139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077216.433139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077216.433139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077216.433224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077216.753235:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077216.945166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077217.457214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077217.457214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077217.457219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077217.457278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077217.777146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077217.969284:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077218.481124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077218.481124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077218.481130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077218.481270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077218.801189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077218.993191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077219.505169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077219.505169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077219.505169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077219.505240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077219.825128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077220.017169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077220.529119:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077220.529124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077220.529158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077220.529240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077220.849156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077220.849283:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077220.849298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077220.849304:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077220.849317:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2926 00000100:00080000:2.0:1769077220.849334:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077220.849344:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077220.849425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913428480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077220.849473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077220.849478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077220.849481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077220.849487:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2926 00000100:00080000:2.0:1769077220.849493:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077220.849500:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077220.849517:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077220.849529:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913428480:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077220.849540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913428608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077220.849558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077220.849563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077220.849566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077220.849573:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2926 00000100:00080000:2.0:1769077220.849578:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077220.849583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077220.849590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077220.849604:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913428736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077220.849619:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077220.849636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077220.849642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077220.849646:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077220.849654:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2926 00000100:00080000:2.0:1769077220.849661:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077220.849668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077220.849676:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077220.849685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077220/real 1769077220] req@ffff935002a61c00 x1855008913428480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077220.849696:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913428864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077220.849710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913428480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077220.849711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077220.849732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077220.849738:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077220.849763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2926 00000100:00080000:0.0:1769077220.849767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077220.849771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077220.849781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077220.849789:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077220.849800:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913428480:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769077220.849820:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077220.849834:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913428992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077220.849841:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913428608:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077220.849861:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077220.849864:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077220.849866:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077220.849889:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2926 00000800:00000100:0.0:1769077220.849901:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077220.849906:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077220.849916:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077220.849921:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077220/real 1769077220] req@ffff935002a61f80 x1855008913428608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077220.849927:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077220.849944:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913428608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077220.849954:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913429120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077220.850054:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077220.850059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077220.850063:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077220.850069:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2926 00000100:00080000:2.0:1769077220.850076:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077220.850078:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077220.850090:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077220.850097:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913428608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077220.850107:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913428736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077220.850151:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077220.850153:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077220.850159:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077220.850168:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077220/real 1769077220] req@ffff935002a62680 x1855008913428736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077220.850177:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913429248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077220.850199:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913428736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077220.850262:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077220.850271:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077220.850275:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913428736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077220.850285:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913428864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077220.850302:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077220.850315:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077220/real 1769077220] req@ffff935002a60a80 x1855008913428864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077220.850325:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913428864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077220.850331:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077220.850333:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077220.850336:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913428864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077220.850350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428992:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077220.850361:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077220.850368:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077220/real 1769077220] req@ffff935002a63100 x1855008913428992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077220.850376:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913428992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077220.850381:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077220.850383:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077220.850386:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913428992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077220.850390:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913429120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077220.850419:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077220.850427:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077220/real 1769077220] req@ffff935002a61180 x1855008913429120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077220.850435:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913429120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077220.850441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077220.850443:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077220.850445:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913429120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077220.850450:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913429248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077220.850461:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077220.850468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077220/real 1769077220] req@ffff935002a61880 x1855008913429248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077220.850476:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913429248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077220.850481:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077220.850482:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077220.850484:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913429248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077221.041158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077221.553208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077221.553208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077221.553208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077221.553293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077221.873178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077222.065123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077222.577122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077222.577126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077222.577170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077222.577219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077222.897162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077223.089136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077223.601125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077223.601130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077223.601131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077223.601245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077223.921176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077224.113157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077224.625160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077224.625166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077224.625209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077224.625216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077224.945214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077225.137201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077225.649148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077225.649148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077225.649153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077225.649181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077225.969171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077225.969174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077225.969183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077225.969187:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077225.969195:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2931 00000100:00080000:2.0:1769077225.969205:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077225.969211:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077225.969245:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913429376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077225.969271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077225.969274:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077225.969280:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077225.969283:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2931 00000100:00080000:2.0:1769077225.969286:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077225.969288:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077225.969296:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077225.969306:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913429504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077225.969329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077225.969332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077225.969334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077225.969337:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2931 00000100:00080000:2.0:1769077225.969340:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077225.969342:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913429376:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077225.969345:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077225.969354:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077225.969364:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913429632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077225.969371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077225.969374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077225.969375:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077225.969377:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2931 00000100:00080000:2.0:1769077225.969379:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077225.969382:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077225.969385:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077225.969391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913429760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077225.969398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077225.969400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077225.969402:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077225.969405:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2931 00000800:00000100:0.0:1769077225.969408:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077225.969409:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077225.969414:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077225.969417:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077225.969427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913429888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077225.969433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077225.969436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077225.969438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077225.969440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2931 00000100:00080000:2.0:1769077225.969442:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077225.969444:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077225.969448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077225.969455:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff935002a62a00 x1855008913429376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077225.969459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913430016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077225.969482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077225.969485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077225.969486:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077225.969489:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913429376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077225.969492:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2931 00000100:00080000:2.0:1769077225.969498:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077225.969504:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077225.969507:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769077225.969508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077225.969515:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077225.969518:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913430144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077225.969520:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913429376:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077225.969549:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913429504:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077225.969574:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077225.969588:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff935002a60000 x1855008913429504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077225.969603:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913429504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077225.969622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077225.969625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077225.969628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913429504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077225.969632:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913429632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077225.969649:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077225.969657:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff935002a62300 x1855008913429632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077225.969672:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913429632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077225.969678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077225.969681:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077225.969683:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913429632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077225.969687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913429760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077225.969698:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077225.969704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff935002a61500 x1855008913429760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077225.969712:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913429760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077225.969718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077225.969720:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077225.969722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913429760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077225.969725:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913429888:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077225.969734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077225.969740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff935002a61880 x1855008913429888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077225.969747:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913429888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077225.969753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077225.969755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077225.969756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913429888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077225.969759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077225.969769:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077225.969775:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff935002a61180 x1855008913430016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077225.969782:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913430016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077225.969787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077225.969789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077225.969791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077225.969795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077225.969804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077225.969810:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff935002a63100 x1855008913430144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077225.969817:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913430144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077225.969822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077225.969823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077225.969825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077226.161171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077226.673128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077226.673128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077226.673169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077226.673204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077226.993169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077227.185132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077227.697122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077227.697130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077227.697152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077227.697165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077228.017158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077228.209155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077228.721175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077228.721229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077228.721230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077228.721261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077229.041213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077229.233127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077229.745174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077229.745176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077229.745189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077229.745230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077230.065152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077230.257264:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077230.769198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077230.769199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077230.769199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077230.769222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077231.089122:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077231.089123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077231.089132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077231.089137:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077231.089146:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2936 00000100:00080000:2.0:1769077231.089156:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077231.089162:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077231.089200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913430272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077231.089252:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077231.089255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077231.089257:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077231.089261:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2936 00000100:00080000:2.0:1769077231.089265:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077231.089269:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077231.089278:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077231.089291:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913430272:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077231.089293:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913430400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077231.089306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077231.089309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077231.089311:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077231.089314:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2936 00000100:00080000:2.0:1769077231.089317:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077231.089319:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077231.089324:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077231.089334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913430528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077231.089340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077231.089343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077231.089344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077231.089346:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2936 00000100:00080000:2.0:1769077231.089349:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077231.089351:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077231.089354:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077231.089362:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913430656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077231.089368:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077231.089375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077231.089378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077231.089380:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077231.089385:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2936 00000100:00080000:2.0:1769077231.089394:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077231.089397:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077231.089401:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077231.089411:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077231/real 1769077231] req@ffff935002a60a80 x1855008913430272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077231.089414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913430784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077231.089439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077231.089442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077231.089445:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913430272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077231.089446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077231.089453:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2936 00000100:00080000:0.0:1769077231.089464:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077231.089466:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077231.089471:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077231.089474:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077231.089481:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077231.089483:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913430272:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077231.089491:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913430912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077231.089499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077231.089501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077231.089502:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077231.089505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2936 00000100:00080000:2.0:1769077231.089507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077231.089509:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077231.089513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077231.089515:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913430400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077231.089523:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913431040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077231.089549:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077231.089561:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077231/real 1769077231] req@ffff935002a62680 x1855008913430400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077231.089570:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913430400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077231.089577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077231.089579:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077231.089582:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913430400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077231.089586:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913430528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077231.089598:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077231.089604:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077231/real 1769077231] req@ffff935002a61f80 x1855008913430528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077231.089612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913430528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077231.089617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077231.089619:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077231.089621:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913430528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077231.089624:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913430656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077231.089634:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077231.089640:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077231/real 1769077231] req@ffff935002a61c00 x1855008913430656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077231.089647:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913430656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077231.089653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077231.089654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077231.089656:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913430656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077231.089659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430784:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077231.089669:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077231.089674:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077231/real 1769077231] req@ffff935002a63100 x1855008913430784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077231.089682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913430784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077231.089687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077231.089688:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077231.089690:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077231.089693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077231.089703:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077231.089709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077231/real 1769077231] req@ffff935002a61180 x1855008913430912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077231.089715:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913430912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077231.089727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077231.089729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077231.089731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913430912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077231.089746:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077231.089756:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077231.089764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077231/real 1769077231] req@ffff935002a61880 x1855008913431040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077231.089781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913431040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077231.089790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077231.089793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077231.089797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077231.281127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077231.793149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077231.793149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077231.793149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077231.793184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077232.113151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077232.305151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077232.817151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077232.817152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077232.817152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077232.817188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077233.137142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077233.329124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077233.841165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077233.841165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077233.841165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077233.841260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077234.161265:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077234.353207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077234.865182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077234.865182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077234.865182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077234.865238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077235.185115:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077235.377149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077235.889138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077235.889139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077235.889163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077235.889190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077236.209165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077236.209177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1769077236.209178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077236.209181:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077236.209193:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2941 00000100:00080000:1.0:1769077236.209204:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077236.209212:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077236.209250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913431168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077236.209278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077236.209282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077236.209284:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077236.209289:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2941 00000100:00080000:1.0:1769077236.209292:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077236.209296:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077236.209306:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077236.209321:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913431296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077236.209328:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913431168:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077236.209340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077236.209344:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077236.209346:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077236.209352:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2941 00000100:00080000:1.0:1769077236.209356:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077236.209360:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077236.209366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077236.209378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913431424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077236.209387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077236.209390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077236.209392:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077236.209396:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2941 00000800:00000100:0.0:1769077236.209402:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077236.209403:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077236.209410:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077236.209415:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077236.209429:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913431552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077236.209438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077236.209441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077236.209443:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077236.209448:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2941 00000100:00080000:1.0:1769077236.209452:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077236.209457:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077236/real 1769077236] req@ffff9350100cad80 x1855008913431168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077236.209459:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077236.209468:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077236.209480:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913431168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077236.209485:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913431680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077236.209496:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077236.209502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077236.209503:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077236.209510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077236.209512:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913431168:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077236.209514:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077236.209521:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2941 00000100:00080000:1.0:1769077236.209527:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077236.209530:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077236.209535:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077236.209536:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913431296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077236.209554:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913431808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077236.209565:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077236.209567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077236.209573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077236.209575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077236.209579:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077236/real 1769077236] req@ffff9350100c8000 x1855008913431296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077236.209584:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2941 00000100:00080000:1.0:1769077236.209593:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077236.209598:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913431296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077236.209600:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077236.209608:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077236.209610:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077236.209616:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077236.209619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913431296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077236.209624:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913431936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077236.209627:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913431424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077236.209652:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077236.209661:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077236/real 1769077236] req@ffff9350100c8a80 x1855008913431424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077236.209669:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913431424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077236.209675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077236.209677:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077236.209679:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913431424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077236.209684:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913431552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077236.209694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077236.209699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077236/real 1769077236] req@ffff9350100c9180 x1855008913431552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077236.209706:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913431552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077236.209712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077236.209713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077236.209715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913431552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077236.209719:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431680:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077236.209729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077236.209735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077236/real 1769077236] req@ffff9350100cb100 x1855008913431680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077236.209742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913431680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077236.209747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077236.209749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077236.209751:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077236.209762:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077236.209771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077236.209777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077236/real 1769077236] req@ffff9350100c9c00 x1855008913431808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077236.209785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913431808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077236.209791:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077236.209792:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077236.209794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077236.209798:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077236.209807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077236.209813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077236/real 1769077236] req@ffff9350100c9f80 x1855008913431936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077236.209821:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913431936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077236.209826:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077236.209827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077236.209829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913431936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077236.401189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077236.913172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077236.913172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077236.913194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077236.913200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077237.233172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077237.425142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077237.937146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077237.937177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077237.937182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077237.937211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077238.257148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077238.449151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077238.961174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077238.961176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077238.961201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077238.961217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077239.281161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077239.473155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077239.985175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077239.985175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077239.985193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077239.985257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077240.305178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077240.497174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077241.009165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077241.009165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077241.009197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077241.009296:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769077241.329184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077241.329187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077241.329199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077241.329205:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077241.329218:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2947 00000100:00080000:1.0:1769077241.329232:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077241.329240:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077241.329289:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913432064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077241.329317:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077241.329320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077241.329321:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077241.329325:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2947 00000100:00080000:1.0:1769077241.329329:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077241.329333:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077241.329344:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077241.329357:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913432192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077241.329363:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913432064:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077241.329376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077241.329379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077241.329380:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077241.329385:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2947 00000100:00080000:1.0:1769077241.329387:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077241.329390:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077241.329394:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077241.329403:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913432320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077241.329410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077241.329413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077241.329415:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077241.329417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2947 00000100:00080000:1.0:1769077241.329431:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077241.329434:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077241.329437:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077241.329445:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913432448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077241.329454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077241.329464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077241.329467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077241.329469:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077241.329472:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2947 00000100:00080000:1.0:1769077241.329474:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077241.329476:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077241.329479:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077241.329485:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913432576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077241.329493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077241.329496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077241.329498:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077241.329501:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2947 00000100:00080000:1.0:1769077241.329504:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077241.329506:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077241.329509:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077241.329518:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913432704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077241.329525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077241.329529:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077241.329531:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077241/real 1769077241] req@ffff9350100ca680 x1855008913432064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077241.329531:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077241.329538:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2947 00000100:00080000:1.0:1769077241.329543:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077241.329549:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077241.329555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077241.329566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913432832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077241.329571:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913432064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077241.329587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077241.329595:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077241.329600:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913432064:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077241.329631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913432192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077241.329659:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077241.329674:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077241/real 1769077241] req@ffff9350100c8e00 x1855008913432192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077241.329689:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913432192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077241.329699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077241.329703:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077241.329707:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913432192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077241.329718:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913432320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077241.329741:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077241.329754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077241/real 1769077241] req@ffff9350100c9880 x1855008913432320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077241.329768:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913432320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077241.329778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077241.329782:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077241.329786:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913432320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077241.329793:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913432448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077241.329815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077241.329827:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077241/real 1769077241] req@ffff9350100c9500 x1855008913432448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077241.329841:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913432448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077241.329850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077241.329854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077241.329857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913432448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077241.329864:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913432576:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077241.329885:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077241.329905:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077241/real 1769077241] req@ffff9350100caa00 x1855008913432576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077241.329919:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913432576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077241.329946:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077241.329950:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077241.329955:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913432576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077241.329962:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913432704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077241.329995:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077241.330009:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077241/real 1769077241] req@ffff9350100c8380 x1855008913432704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077241.330023:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913432704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077241.330032:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077241.330037:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077241.330041:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913432704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077241.330048:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913432832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077241.330141:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077241.330153:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077241/real 1769077241] req@ffff9350100ca300 x1855008913432832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077241.330167:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913432832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077241.330184:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077241.330188:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077241.330192:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913432832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077241.521244:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077242.033161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077242.033178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077242.033208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077242.033208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077242.353173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077242.545187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077243.057155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077243.057168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077243.057191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077243.057215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077243.377141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077243.569163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077244.081119:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077244.081187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077244.081225:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077244.081245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077244.401165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077244.593123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077245.105144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077245.105164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077245.105180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077245.105206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077245.425133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077245.617213:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077246.129178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077246.129178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077246.129311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077246.129379:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077246.449144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077246.449188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077246.449197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077246.449202:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077246.449212:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2952 00000100:00080000:2.0:1769077246.449224:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077246.449230:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077246.449275:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913432960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077246.449303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077246.449306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077246.449307:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077246.449310:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2952 00000100:00080000:2.0:1769077246.449313:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077246.449315:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077246.449325:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077246.449338:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913433088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077246.449346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077246.449350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913432960:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077246.449351:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077246.449357:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077246.449369:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2952 00000100:00080000:2.0:1769077246.449371:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077246.449375:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077246.449383:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077246.449400:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913433216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077246.449410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077246.449414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077246.449415:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077246.449421:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2952 00000100:00080000:2.0:1769077246.449426:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077246.449432:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077246.449434:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769077246.449447:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077246.449465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913433344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077246.449479:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077246/real 1769077246] req@ffff935002a61500 x1855008913432960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077246.449480:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077246.449501:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077246.449509:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077246.449519:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913432960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077246.449530:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2952 00000100:00080000:0.0:1769077246.449532:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077246.449538:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077246.449539:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077246.449548:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913432960:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077246.449554:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077246.449564:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077246.449575:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077246.449585:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913433472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077246.449600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077246.449604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1769077246.449609:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077246.449611:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077246.449626:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2952 00000100:00100000:0.0:1769077246.449627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077246/real 1769077246] req@ffff935002a62300 x1855008913433088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077246.449638:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077246.449645:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913433088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077246.449649:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077246.449666:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077246.449668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077246.449680:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077246.449682:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077246.449689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077246.449700:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913433600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077246.449711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077246.449722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077246/real 1769077246] req@ffff935002a60000 x1855008913433216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077246.449725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077246.449737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077246.449744:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077246.449751:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913433216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077246.449756:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2952 00000100:00080000:2.0:1769077246.449770:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077246.449772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077246.449780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077246.449782:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077246.449786:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433216:192.168.204.121@tcp:8:kworker.0 00000008:00100000:2.0:1769077246.449798:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077246.449799:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077246.449827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913433728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077246.449828:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077246.449839:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077246/real 1769077246] req@ffff935002a62a00 x1855008913433344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077246.449867:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913433344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077246.449877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077246.449886:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077246.449891:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077246.449912:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913433472:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077246.449929:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077246.449938:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077246/real 1769077246] req@ffff935002a61500 x1855008913433472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077246.449947:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913433472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077246.449953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077246.449955:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077246.449957:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913433472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077246.449962:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913433600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077246.449991:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077246.450002:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077246/real 1769077246] req@ffff935002a61880 x1855008913433600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077246.450015:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913433600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077246.450023:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077246.450027:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077246.450030:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913433600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077246.450036:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913433728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077246.450082:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077246.450092:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077246/real 1769077246] req@ffff935002a61180 x1855008913433728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077246.450100:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913433728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077246.450106:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077246.450108:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077246.450110:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913433728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077246.641128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077247.153163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077247.153183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077247.153204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077247.153205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077247.473156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077247.665184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077248.177134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077248.177148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077248.177168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077248.177181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077248.497127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077248.689121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077249.201152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077249.201152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077249.201152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077249.201223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077249.521115:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077249.713132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077250.225163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077250.225163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077250.225163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077250.225200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077250.545122:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077250.737134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077251.249143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077251.249144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077251.249156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077251.249185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077251.569160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077251.569167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077251.569170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077251.569177:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077251.569189:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2957 00000100:00080000:3.0:1769077251.569200:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077251.569206:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077251.569240:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913433856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077251.569261:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077251.569264:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077251.569265:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077251.569268:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2957 00000100:00080000:3.0:1769077251.569271:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077251.569273:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077251.569282:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077251.569290:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913433984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077251.569298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077251.569301:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077251.569304:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077251.569305:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913433856:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077251.569309:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2957 00000100:00080000:3.0:1769077251.569313:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077251.569318:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077251.569321:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077251.569329:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913434112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077251.569335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077251.569338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077251.569339:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077251.569342:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2957 00000100:00080000:3.0:1769077251.569344:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077251.569346:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077251.569348:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077251.569355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913434240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077251.569361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077251.569363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077251.569365:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077251.569368:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2957 00000100:00080000:3.0:1769077251.569371:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077251.569373:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077251.569376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077251.569384:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913434368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077251.569390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077251.569393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077251.569394:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077251.569396:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2957 00000100:00080000:3.0:1769077251.569399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077251.569402:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077251.569406:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077251.569414:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913434496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077251.569421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077251.569424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077251.569426:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433984:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077251.569427:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077251.569436:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2957 00000100:00080000:3.0:1769077251.569440:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077251.569442:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077251.569446:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077251.569458:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913434624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077251.569481:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913434112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077251.569511:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913434240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077251.569535:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913434368:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077251.569563:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913434496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077251.569587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913434624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077251.761141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077252.273162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077252.273162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077252.273162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077252.273203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077252.593130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077252.785107:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077253.297135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077253.297135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077253.297153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077253.297158:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077253.617182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077253.809194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077254.321148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077254.321148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077254.321149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077254.321184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077254.641127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:2.0:1769077254.641301:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:2.0:1769077254.641334:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077254.641350:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077254.641382:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077254.641390:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077254.641404:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077254.641416:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:2.0:1769077254.641427:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077254.641433:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077251/real 1769077254] req@ffff935002a7f100 x1855008913433984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1769077254.641437:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077254.641466:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913433984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077254.641479:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077254.641485:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077254.641489:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913433984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077254.641497:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077251/real 1769077254] req@ffff935002a7c380 x1855008913434112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077254.641513:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913434112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077254.641522:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077254.641526:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077254.641530:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913434112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077254.641537:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077251/real 1769077254] req@ffff935002a7ca80 x1855008913434240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077254.641551:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913434240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077254.641559:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077254.641562:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077254.641566:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913434240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077254.641572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077251/real 1769077254] req@ffff935002a7e300 x1855008913434368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077254.641585:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913434368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077254.641594:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077254.641597:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077254.641601:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913434368:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077254.641609:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077251/real 1769077254] req@ffff935002a7c700 x1855008913433856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077254.641624:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913433856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077254.641632:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077254.641636:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077254.641640:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913433856:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077254.641685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077251/real 1769077254] req@ffff935002a7f800 x1855008913434496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077254.641700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913434496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077254.641709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077254.641712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077254.641716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913434496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077254.641724:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077251/real 1769077254] req@ffff935002a7c000 x1855008913434624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077254.641737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913434624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077254.641745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077254.641749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077254.641752:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913434624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077254.833129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077255.345127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077255.345127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077255.345145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077255.345187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077255.665144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077255.857146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077256.369170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077256.369170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077256.369170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077256.369204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077256.689109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077256.689118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077256.689120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077256.689125:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077256.689135:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2962 00000100:00080000:3.0:1769077256.689145:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077256.689159:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077256.689193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913434752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077256.689216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077256.689219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077256.689220:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077256.689223:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2962 00000100:00080000:3.0:1769077256.689226:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077256.689229:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077256.689238:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077256.689249:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913434752:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077256.689250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913434880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077256.689267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077256.689272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077256.689274:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077256.689279:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2962 00000100:00080000:3.0:1769077256.689283:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077256.689287:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077256.689294:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077256.689311:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913435008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077256.689322:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077256.689324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077256.689330:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077256.689332:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077256.689338:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2962 00000100:00080000:3.0:1769077256.689343:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077256.689348:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077256.689354:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077256.689373:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077256/real 1769077256] req@ffff935002a7fb80 x1855008913434752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077256.689378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913435136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077256.689391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077256.689398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077256.689402:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077256.689407:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2962 00000100:00080000:3.0:1769077256.689412:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077256.689420:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913434752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077256.689434:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077256.689436:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077256.689441:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769077256.689446:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077256.689448:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913434752:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077256.689465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913435264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077256.689477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077256.689481:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077256.689484:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913434880:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077256.689485:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077256.689496:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2962 00000100:00080000:3.0:1769077256.689502:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077256.689507:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077256.689513:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077256.689522:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077256.689530:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913435392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077256.689541:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077256/real 1769077256] req@ffff935002a7df80 x1855008913434880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077256.689545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077256.689554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077256.689559:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077256.689566:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2962 00000100:00100000:0.0:1769077256.689570:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913434880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077256.689572:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077256.689581:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077256.689589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769077256.689591:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077256.689597:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077256.689603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913434880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077256.689610:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913435520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077256.689614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077256.689644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077256.689660:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077256/real 1769077256] req@ffff935002a7d180 x1855008913435008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077256.689675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913435008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077256.689684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077256.689688:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077256.689692:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077256.689699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077256.689720:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077256.689729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077256/real 1769077256] req@ffff935002a7dc00 x1855008913435136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077256.689739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913435136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077256.689749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077256.689753:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077256.689757:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077256.689763:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913435264:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077256.689785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077256.689814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077256/real 1769077256] req@ffff935002a7c000 x1855008913435264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077256.689832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913435264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077256.689843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077256.689846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077256.689850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913435264:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077256.689870:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913435392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077256.689891:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077256.689905:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077256/real 1769077256] req@ffff935002a7f800 x1855008913435392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077256.689924:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913435392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077256.689932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077256.689936:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077256.689939:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913435392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077256.689945:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913435520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077256.689962:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077256.689974:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077256/real 1769077256] req@ffff935002a7c700 x1855008913435520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077256.689986:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913435520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077256.690000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077256.690003:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077256.690007:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913435520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077256.881139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077257.393121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077257.393128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077257.393130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077257.393164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077257.713096:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077257.905104:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077258.417186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077258.417186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077258.417192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077258.417234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077258.737155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077258.929184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077259.441158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077259.441182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077259.441208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077259.441216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077259.761173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077259.953196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077260.465179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077260.465179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077260.465179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077260.465215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077260.785145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077260.977172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077261.489125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077261.489125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077261.489146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077261.489162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077261.809149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077261.809159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077261.809164:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077261.809172:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2967 00000100:00080000:0.0:1769077261.809182:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077261.809188:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077261.809223:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913435648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077261.809249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077261.809252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077261.809253:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077261.809257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2967 00000100:00080000:0.0:1769077261.809259:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077261.809263:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077261.809272:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077261.809283:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913435776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077261.809307:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077261.809311:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913435648:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077261.809312:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077261.809315:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077261.809324:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2967 00000100:00080000:0.0:1769077261.809326:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077261.809328:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077261.809332:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077261.809342:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913435904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077261.809349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077261.809352:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077261.809353:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077261.809356:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2967 00000100:00080000:0.0:1769077261.809359:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077261.809361:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077261.809387:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077261.809397:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077261.809401:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913436032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077261.809409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077261.809412:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077261.809414:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077261.809416:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2967 00000100:00080000:0.0:1769077261.809419:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077261.809421:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077261.809424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077261.809434:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913436160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077261.809440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077261.809443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077261.809444:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077261.809450:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2967 00000100:00100000:2.0:1769077261.809452:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077261/real 1769077261] req@ffff9350103fad80 x1855008913435648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077261.809453:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077261.809460:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077261.809484:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077261.809495:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913435648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077261.809502:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913436288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077261.809506:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077261.809516:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077261.809522:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913435648:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077261.809523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077261.809529:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077261.809532:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077261.809537:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2967 00000100:00100000:2.0:1769077261.809548:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435776:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077261.809550:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077261.809556:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077261.809562:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077261.809589:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077261.809594:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913436416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077261.809605:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077261/real 1769077261] req@ffff9350103fb800 x1855008913435776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077261.809617:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913435776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077261.809626:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077261.809626:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077261.809649:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077261.809653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077261.809659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077261.809675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077261.809681:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077261/real 1769077261] req@ffff9350103f9f80 x1855008913435904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077261.809691:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913435904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077261.809700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077261.809703:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077261.809707:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913435904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077261.809725:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077261.809746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077261.809759:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077261/real 1769077261] req@ffff9350103f9180 x1855008913436032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077261.809772:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913436032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077261.809778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077261.809780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077261.809783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077261.809787:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913436160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077261.809801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077261.809808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077261/real 1769077261] req@ffff9350103f9500 x1855008913436160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077261.809816:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913436160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077261.809822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077261.809824:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077261.809826:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913436160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077261.809829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913436288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077261.809839:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077261.809845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077261/real 1769077261] req@ffff9350103fbb80 x1855008913436288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077261.809852:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913436288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077261.809857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077261.809859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077261.809861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913436288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077261.809873:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913436416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077261.809883:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077261.809889:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077261/real 1769077261] req@ffff9350103f9c00 x1855008913436416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077261.809896:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913436416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077261.809919:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077261.809921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077261.809923:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913436416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077262.001134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077262.513160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077262.513160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077262.513160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077262.513192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077262.833166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077263.025180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077263.537159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077263.537172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077263.537230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077263.537433:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077263.857173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077264.049155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077264.561173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077264.561180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077264.561180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077264.561219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077264.881206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077265.073166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077265.585127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077265.585127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077265.585157:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077265.585168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077265.905161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077266.097139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077266.609188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077266.609201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077266.609250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077266.609267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077266.929235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.929245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077266.929249:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.929257:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2972 00000100:00080000:0.0:1769077266.929268:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.929275:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077266.929275:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077266.929314:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913436544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.929346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.929349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077266.929350:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.929353:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2972 00000100:00080000:0.0:1769077266.929356:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.929359:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077266.929368:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077266.929377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913436672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077266.929382:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913436544:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077266.929388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.929392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077266.929393:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.929396:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2972 00000100:00080000:0.0:1769077266.929397:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.929399:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077266.929403:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077266.929413:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913436800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.929420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.929422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077266.929423:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.929425:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2972 00000100:00080000:0.0:1769077266.929427:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.929428:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077266.929431:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077266.929439:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913436928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.929444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.929446:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077266.929448:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.929451:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2972 00000100:00080000:0.0:1769077266.929454:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.929457:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077266.929461:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077266.929461:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077266.929472:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913437056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.929479:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.929482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077266.929483:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.929486:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2972 00000100:00080000:0.0:1769077266.929488:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.929490:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077266.929520:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077266.929531:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9350103fb100 x1855008913436544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077266.929538:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913437184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.929555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077266.929557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913436544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077266.929563:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077266.929569:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077266.929578:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077266.929580:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2972 00000100:00080000:2.0:1769077266.929586:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077266.929590:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077266.929593:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913436544:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077266.929613:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077266.929620:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436672:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769077266.929621:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077266.929694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077266.929697:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913437312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077266.929708:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9350103f8700 x1855008913436672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077266.929739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913436672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077266.929747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077266.929749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077266.929754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077266.929760:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077266.929774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077266.929781:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9350103f8380 x1855008913436800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077266.929789:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913436800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077266.929812:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077266.929814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077266.929817:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077266.929822:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077266.929835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077266.929842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9350103fb480 x1855008913436928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077266.929850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913436928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077266.929857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077266.929859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077266.929862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913436928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077266.929866:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437056:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077266.929888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077266.929895:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9350103faa00 x1855008913437056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077266.929904:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913437056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077266.929911:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077266.929913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077266.929915:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077266.929919:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077266.929930:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077266.929946:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9350103f9c00 x1855008913437184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077266.929955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913437184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077266.929961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077266.929963:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077266.929965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077266.929996:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077266.930016:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077266.930022:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9350103fbb80 x1855008913437312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077266.930028:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913437312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077266.930034:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077266.930052:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077266.930054:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077267.121167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077267.633208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077267.633229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077267.633250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077267.633339:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077267.953273:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077268.145149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077268.657196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077268.657198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077268.657268:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077268.657298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077268.977220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077269.169176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077269.681193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077269.681205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077269.681257:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077269.681293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077270.001276:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077270.193141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077270.705202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077270.705210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077270.705249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077270.705291:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077271.025190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077271.217223:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077271.729173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077271.729173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077271.729173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077271.729277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077272.049215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077272.049217:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077272.049233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077272.049240:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077272.049258:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2977 00000100:00080000:0.0:1769077272.049275:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077272.049287:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077272.049366:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913437440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077272.049410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077272.049418:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077272.049421:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077272.049430:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2977 00000100:00080000:0.0:1769077272.049439:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077272.049447:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077272.049461:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077272.049468:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913437440:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077272.049488:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913437568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077272.049504:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077272.049509:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077272.049511:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077272.049516:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2977 00000100:00080000:0.0:1769077272.049521:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077272.049525:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077272.049532:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077272.049564:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913437696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077272.049603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077272.049608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077272.049610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077272.049622:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2977 00000800:00000100:2.0:1769077272.049625:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077272.049630:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077272.049639:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077272.049647:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077272.049665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913437824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077272.049688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077272.049693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077272.049696:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077272.049702:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2977 00000100:00100000:2.0:1769077272.049709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077272/real 1769077272] req@ffff9350103f9500 x1855008913437440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077272.049711:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077272.049743:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077272.049754:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077272.049772:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913437440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077272.049774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913437952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077272.049801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077272.049801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077272.049812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077272.049816:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077272.049817:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077272.049828:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2977 00000100:00100000:2.0:1769077272.049830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913437440:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077272.049834:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077272.049844:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077272.049849:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077272.049862:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913438080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077272.049873:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913437568:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077272.049878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077272.049884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077272.049887:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077272.049893:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2977 00000100:00080000:0.0:1769077272.049898:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077272.049902:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077272.049909:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077272.049914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077272.049945:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077272/real 1769077272] req@ffff9350103f9180 x1855008913437568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077272.049950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913438208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077272.049991:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913437568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077272.050003:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077272.050008:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077272.050013:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913437568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077272.050023:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913437696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077272.050095:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077272.050108:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077272/real 1769077272] req@ffff9350103f9f80 x1855008913437696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077272.050121:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913437696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077272.050130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077272.050133:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077272.050137:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913437696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077272.050143:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913437824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077272.050171:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077272.050180:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077272/real 1769077272] req@ffff9350103fb800 x1855008913437824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077272.050191:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913437824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077272.050199:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077272.050202:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077272.050205:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913437824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077272.050211:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077272.050225:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077272.050235:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077272/real 1769077272] req@ffff9350103fad80 x1855008913437952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077272.050246:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913437952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077272.050255:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077272.050258:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077272.050261:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913437952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077272.050280:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077272.050295:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077272.050304:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077272/real 1769077272] req@ffff9350103fbb80 x1855008913438080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077272.050315:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913438080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077272.050335:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077272.050338:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077272.050341:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077272.050347:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077272.050382:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077272.050393:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077272/real 1769077272] req@ffff9350103f9c00 x1855008913438208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077272.050406:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913438208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077272.050415:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077272.050418:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077272.050422:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077272.241152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077272.753175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077272.753238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077272.753293:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077272.753390:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077273.073246:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077273.265177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077273.777283:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077273.777283:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077273.777313:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077273.777408:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077274.097190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077274.289211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077274.801174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077274.801185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077274.801216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077274.801260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077275.121312:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077275.313203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077275.825171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077275.825172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077275.825204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077275.825231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077276.145206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077276.337218:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077276.849133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077276.849188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077276.849221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077276.849225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769077277.169227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077277.169233:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077277.169244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077277.169250:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077277.169262:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2982 00000100:00080000:3.0:1769077277.169279:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077277.169288:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077277.169352:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913438336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077277.169381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077277.169384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077277.169386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077277.169389:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2982 00000100:00080000:3.0:1769077277.169392:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077277.169395:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077277.169406:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077277.169416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913438464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077277.169435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077277.169438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077277.169442:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077277.169443:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913438336:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077277.169447:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2982 00000100:00080000:3.0:1769077277.169467:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077277.169470:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077277.169475:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077277.169484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913438592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077277.169492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077277.169495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077277.169497:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077277.169500:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2982 00000100:00080000:3.0:1769077277.169502:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077277.169505:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077277.169508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077277.169516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913438720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077277.169522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077277.169525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077277.169526:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077277.169529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2982 00000100:00080000:3.0:1769077277.169532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077277.169534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077277.169537:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077277.169542:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077277.169548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913438848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077277.169556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077277.169559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077277.169561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077277.169564:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2982 00000100:00080000:3.0:1769077277.169566:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077277.169568:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077277.169573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077277.169581:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913438976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077277.169587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077277.169589:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077277/real 1769077277] req@ffff935002a7e300 x1855008913438336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077277.169590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077277.169595:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077277.169604:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2982 00000100:00080000:3.0:1769077277.169607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077277.169610:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077277.169614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077277.169615:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913438336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077277.169623:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913439104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077277.169625:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077277.169634:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077277.169640:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913438336:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077277.169663:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913438464:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077277.169679:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077277.169688:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077277/real 1769077277] req@ffff935002a7ca80 x1855008913438464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077277.169697:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913438464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077277.169708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077277.169710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077277.169713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913438464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077277.169717:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913438592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077277.169728:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077277.169734:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077277/real 1769077277] req@ffff935002a7c380 x1855008913438592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077277.169741:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913438592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077277.169746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077277.169748:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077277.169750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913438592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077277.169753:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913438720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077277.169763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077277.169768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077277/real 1769077277] req@ffff935002a7f100 x1855008913438720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077277.169776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913438720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077277.169780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077277.169782:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077277.169785:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913438720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077277.169788:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438848:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077277.169798:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077277.169803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077277/real 1769077277] req@ffff935002a7c700 x1855008913438848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077277.169819:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913438848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077277.169825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077277.169826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077277.169828:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077277.169832:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077277.169841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077277.169846:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077277/real 1769077277] req@ffff935002a7f800 x1855008913438976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077277.169858:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913438976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077277.169866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077277.169870:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077277.169881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913438976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077277.169888:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913439104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077277.169908:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077277.169920:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077277/real 1769077277] req@ffff935002a7c000 x1855008913439104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077277.169928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913439104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077277.169944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077277.169946:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077277.169948:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913439104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077277.361211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077277.873185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077277.873189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077277.873218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077277.873218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077278.193151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077278.385228:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077278.897184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077278.897186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077278.897269:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077278.897321:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077279.217166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077279.409196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077279.921176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077279.921221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077279.921296:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077279.921376:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077280.241210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077280.433171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077280.945173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077280.945174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077280.945212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077280.945313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077281.265157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077281.457213:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077281.969164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077281.969177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077281.969215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077281.969261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077282.289192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077282.289218:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077282.289228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077282.289234:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077282.289246:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2987 00000100:00080000:2.0:1769077282.289261:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077282.289269:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077282.289317:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913439232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077282.289352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077282.289356:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077282.289358:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077282.289363:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2987 00000100:00080000:2.0:1769077282.289396:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077282.289402:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077282.289412:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913439232:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769077282.289416:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077282.289433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913439360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077282.289446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077282.289450:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077282.289452:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077282.289456:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2987 00000100:00080000:2.0:1769077282.289460:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077282.289463:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077282.289468:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077282.289479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913439488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077282.289489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077282.289492:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077282.289494:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077282.289497:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2987 00000100:00080000:2.0:1769077282.289501:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077282.289504:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077282.289508:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077282.289512:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077282.289522:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913439616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077282.289555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077282.289559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077282.289561:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077282.289566:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2987 00000100:00080000:2.0:1769077282.289571:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077282.289576:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077282.289582:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077282.289586:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077282/real 1769077282] req@ffff935002a63100 x1855008913439232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077282.289599:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913439744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077282.289612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077282.289616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077282.289619:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913439232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077282.289620:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077282.289632:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2987 00000100:00080000:2.0:1769077282.289639:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077282.289647:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077282.289649:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769077282.289652:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077282.289659:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077282.289667:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913439232:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077282.289671:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913439872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077282.289685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077282.289690:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077282.289693:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077282.289701:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2987 00000100:00080000:2.0:1769077282.289705:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077282.289712:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077282.289715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913439360:192.168.204.121@tcp:38:kworker.0 00000008:00100000:2.0:1769077282.289725:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077282.289750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913440000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077282.289757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077282.289777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077282/real 1769077282] req@ffff935002a61c00 x1855008913439360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077282.289793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913439360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077282.289802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077282.289807:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077282.289810:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913439360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077282.289817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913439488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077282.289834:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077282.289844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077282/real 1769077282] req@ffff935002a61f80 x1855008913439488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077282.289855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913439488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077282.289872:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077282.289875:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077282.289878:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913439488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077282.289885:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913439616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077282.289902:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077282.289912:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077282/real 1769077282] req@ffff935002a62680 x1855008913439616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077282.289923:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913439616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077282.289931:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077282.289935:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077282.289938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913439616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077282.289943:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913439744:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077282.289957:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077282.289965:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077282/real 1769077282] req@ffff935002a60a80 x1855008913439744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077282.289976:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913439744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077282.289984:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077282.289986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077282.289989:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913439744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077282.289994:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913439872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077282.290007:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077282.290016:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077282/real 1769077282] req@ffff935002a61180 x1855008913439872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077282.290027:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913439872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077282.290035:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077282.290037:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077282.290040:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913439872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077282.290088:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077282.290127:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077282.290141:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077282/real 1769077282] req@ffff935002a61880 x1855008913440000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077282.290155:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913440000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077282.290165:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077282.290169:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077282.290172:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077282.481177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077282.993145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077282.993191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077282.993191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077282.993238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077283.313157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077283.505169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077284.017169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077284.017169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077284.017169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077284.017232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077284.337218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077284.529189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077285.041190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077285.041234:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077285.041235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077285.041266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077285.361209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077285.553165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077286.065185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077286.065239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077286.065242:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077286.065255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077286.385213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077286.577160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077287.089133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077287.089139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077287.089246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077287.089354:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077287.409216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077287.409219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077287.409239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077287.409245:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077287.409256:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2993 00000100:00080000:3.0:1769077287.409269:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077287.409277:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077287.409330:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913440128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077287.409361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077287.409366:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077287.409368:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077287.409373:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2993 00000100:00080000:3.0:1769077287.409377:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077287.409382:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077287.409393:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077287.409407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913440256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077287.409420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077287.409423:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913440128:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077287.409426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077287.409437:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077287.409444:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2993 00000100:00080000:3.0:1769077287.409448:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077287.409454:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077287.409463:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077287.409482:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913440384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077287.409498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077287.409502:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077287.409504:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077287.409508:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2993 00000100:00080000:3.0:1769077287.409522:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077287.409526:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077287.409531:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077287.409534:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077287.409560:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913440512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077287.409574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077287.409577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077287.409580:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077287.409585:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2993 00000100:00080000:3.0:1769077287.409591:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077287.409598:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077287.409604:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077287.409607:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077287/real 1769077287] req@ffff935002a7dc00 x1855008913440128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077287.409630:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913440640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077287.409638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913440128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077287.409681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077287.409683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077287.409689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077287.409696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077287.409700:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913440128:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077287.409704:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077287.409714:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2993 00000100:00080000:3.0:1769077287.409722:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077287.409727:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077287.409734:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077287.409739:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913440256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077287.409762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913440768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077287.409774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077287.409794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077287.409800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077287/real 1769077287] req@ffff935002a7d180 x1855008913440256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077287.409802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077287.409810:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077287.409834:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2993 00000100:00100000:0.0:1769077287.409835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913440256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077287.409851:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077287.409862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077287.409865:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077287.409872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769077287.409878:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077287.409880:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913440256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077287.409896:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913440384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077287.409903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913440896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077287.409931:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077287.409952:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077287/real 1769077287] req@ffff935002a7df80 x1855008913440384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077287.409966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913440384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077287.409975:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077287.409979:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077287.409983:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913440384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077287.409989:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913440512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077287.410012:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077287.410030:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077287/real 1769077287] req@ffff935002a7fb80 x1855008913440512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077287.410089:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913440512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077287.410107:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077287.410113:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077287.410119:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913440512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077287.410131:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440640:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077287.410188:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077287.410211:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077287/real 1769077287] req@ffff935002a7c000 x1855008913440640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077287.410234:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913440640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077287.410262:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077287.410269:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077287.410276:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077287.410356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077287.410384:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077287.410397:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077287/real 1769077287] req@ffff935002a7f800 x1855008913440768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077287.410410:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913440768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077287.410420:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077287.410423:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077287.410427:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077287.410433:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077287.410450:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077287.410460:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077287/real 1769077287] req@ffff935002a7c700 x1855008913440896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077287.410472:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913440896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077287.410480:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077287.410483:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077287.410487:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913440896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077287.601127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077288.113178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077288.113231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077288.113255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077288.113289:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077288.433167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077288.625195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077289.137185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077289.137185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077289.137185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077289.137308:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077289.457174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077289.649113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077290.161132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077290.161136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077290.161213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077290.161277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077290.481201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077290.673152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077291.185157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077291.185173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077291.185174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077291.185253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077291.505191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077291.697137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077292.209186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077292.209219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077292.209291:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077292.209323:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077292.529262:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077292.529277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077292.529285:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077292.529300:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 2998 00000100:00080000:0.0:1769077292.529316:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077292.529328:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077292.529340:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077292.529385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913441024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077292.529425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077292.529430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077292.529433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077292.529440:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2998 00000100:00080000:0.0:1769077292.529446:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077292.529452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077292.529468:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077292.529488:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913441024:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077292.529489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913441152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077292.529541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077292.529548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077292.529552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077292.529561:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2998 00000100:00080000:0.0:1769077292.529584:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077292.529593:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077292.529604:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077292.529627:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077292.529635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913441280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077292.529651:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077292.529656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077292.529659:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077292.529665:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 2998 00000100:00080000:0.0:1769077292.529671:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077292.529677:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077292.529685:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077292.529694:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077292/real 1769077292] req@ffff9350103faa00 x1855008913441024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077292.529710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913441408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077292.529728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913441024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077292.529737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077292.529753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077292.529754:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077292.529761:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077292.529765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077292.529782:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2998 00000100:00100000:2.0:1769077292.529783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913441024:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077292.529800:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077292.529828:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077292.529839:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077292.529850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913441152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077292.529875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913441536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077292.529892:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077292.529902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077292.529910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077292.529914:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077292.529922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077292/real 1769077292] req@ffff9350103fb480 x1855008913441152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077292.529927:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2998 00000100:00080000:0.0:1769077292.529940:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077292.529953:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077292.529961:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913441152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077292.529964:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077292.530013:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077292.530020:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913441664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077292.530022:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077292.530033:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913441152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077292.530054:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913441280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077292.530086:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077292.530115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077292.530138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077292.530141:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077292.530149:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 2998 00000100:00080000:0.0:1769077292.530157:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077292.530163:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077292.530172:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077292.530183:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077292/real 1769077292] req@ffff9350103f8380 x1855008913441280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077292.530239:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913441792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077292.530282:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913441280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077292.530312:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077292.530320:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077292.530325:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913441280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077292.530334:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913441408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077292.530365:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077292.530380:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077292/real 1769077292] req@ffff9350103f8700 x1855008913441408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077292.530394:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913441408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077292.530404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077292.530408:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077292.530414:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913441408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077292.530437:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913441536:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077292.530458:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077292.530469:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077292/real 1769077292] req@ffff9350103fb100 x1855008913441536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077292.530484:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913441536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077292.530494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077292.530498:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077292.530502:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913441536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077292.530509:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913441664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077292.530549:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077292.530559:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077292/real 1769077292] req@ffff9350103faa00 x1855008913441664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077292.530572:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913441664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077292.530581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077292.530584:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077292.530587:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913441664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077292.530593:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913441792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077292.530608:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077292.530635:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077292/real 1769077292] req@ffff9350103f9c00 x1855008913441792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077292.530650:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913441792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077292.530659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077292.530663:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077292.530666:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913441792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077292.721249:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077293.233160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077293.233173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077293.233195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077293.233259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077293.553236:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077293.745134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077294.257141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077294.257141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077294.257158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077294.257240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077294.577146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077294.769137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077295.281164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077295.281167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077295.281174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077295.281266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077295.601125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077295.793179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077296.305148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077296.305149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077296.305166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077296.305249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077296.625148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077296.817147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077297.329165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077297.329165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077297.329165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077297.329272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077297.649147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077297.649147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077297.649156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077297.649159:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077297.649169:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3003 00000100:00080000:1.0:1769077297.649181:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077297.649189:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077297.649230:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913441920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077297.649252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077297.649255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077297.649257:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077297.649260:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3003 00000100:00080000:1.0:1769077297.649262:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077297.649265:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077297.649272:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077297.649282:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913442048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077297.649289:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077297.649292:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077297.649294:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077297.649297:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3003 00000100:00080000:1.0:1769077297.649299:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077297.649301:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077297.649305:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077297.649312:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913442176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077297.649323:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077297.649325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077297.649327:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077297.649330:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3003 00000100:00080000:1.0:1769077297.649333:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077297.649338:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077297.649342:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077297.649342:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913441920:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077297.649360:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913442304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077297.649368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077297.649371:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077297.649373:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077297.649377:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3003 00000100:00080000:1.0:1769077297.649379:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077297.649382:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077297.649385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077297.649392:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913442432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077297.649399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077297.649401:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077297.649402:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077297.649405:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3003 00000100:00080000:1.0:1769077297.649408:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077297.649410:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077297.649413:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077297.649421:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913442560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077297.649427:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077297.649430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077297.649437:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077297.649439:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077297.649443:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3003 00000100:00080000:1.0:1769077297.649448:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077297.649450:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077297.649454:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077297.649463:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913442688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077297.649498:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077297/real 1769077297] req@ffff9350100cb800 x1855008913441920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077297.649515:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913441920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077297.649524:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077297.649528:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077297.649531:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913441920:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077297.649538:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442048:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077297.649553:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077297.649561:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077297/real 1769077297] req@ffff9350100ca300 x1855008913442048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077297.649569:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913442048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077297.649580:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077297.649582:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077297.649585:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077297.649608:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077297.649620:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077297.649626:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077297/real 1769077297] req@ffff9350100c8380 x1855008913442176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077297.649633:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913442176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077297.649638:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077297.649640:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077297.649642:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077297.649646:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077297.649655:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077297.649661:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077297/real 1769077297] req@ffff9350100caa00 x1855008913442304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077297.649668:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913442304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077297.649673:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077297.649674:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077297.649677:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077297.649681:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913442432:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077297.649691:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077297.649696:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077297/real 1769077297] req@ffff9350100c9500 x1855008913442432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077297.649703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913442432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077297.649708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077297.649710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077297.649712:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913442432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077297.649716:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913442560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077297.649725:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077297.649730:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077297/real 1769077297] req@ffff9350100c9880 x1855008913442560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077297.649737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913442560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077297.649742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077297.649744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077297.649746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913442560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077297.649750:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913442688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077297.649759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077297.649768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077297/real 1769077297] req@ffff9350100c8e00 x1855008913442688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077297.649790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913442688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077297.649799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077297.649803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077297.649807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913442688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077297.841195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077298.353156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077298.353176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077298.353181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077298.353313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077298.673225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077298.865170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077299.377184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077299.377184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077299.377190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077299.377258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077299.697166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077299.889166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077300.401175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077300.401176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077300.401176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077300.401297:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077300.721141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077300.913182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077301.425180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077301.425180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077301.425182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077301.425252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077301.745166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077301.937180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077302.449131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077302.449131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077302.449141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077302.449257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077302.769171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077302.769176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077302.769186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077302.769190:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077302.769200:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3008 00000100:00080000:2.0:1769077302.769212:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077302.769220:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077302.769257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913442816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077302.769282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077302.769285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077302.769287:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077302.769290:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3008 00000100:00080000:2.0:1769077302.769293:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077302.769296:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077302.769305:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077302.769317:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913442944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077302.769319:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913442816:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077302.769333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077302.769336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077302.769338:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077302.769341:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3008 00000100:00080000:2.0:1769077302.769344:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077302.769346:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077302.769350:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077302.769358:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913443072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077302.769365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077302.769367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077302.769369:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077302.769371:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3008 00000100:00080000:2.0:1769077302.769373:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077302.769376:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077302.769379:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077302.769386:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913443200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077302.769391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077302.769394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077302.769397:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077302.769399:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077302.769403:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3008 00000100:00080000:2.0:1769077302.769407:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077302.769410:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077302.769413:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077302.769423:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913443328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077302.769429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077302.769431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077302.769432:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077302.769435:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3008 00000100:00080000:2.0:1769077302.769437:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077302.769440:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077302.769444:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077302.769453:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077302/real 1769077302] req@ffff935002a61500 x1855008913442816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077302.769454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913443456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077302.769484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077302.769487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077302.769488:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077302.769492:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3008 00000100:00080000:2.0:1769077302.769494:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077302.769499:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077302.769500:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913442816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077302.769502:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077302.769513:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913443584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077302.769518:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077302.769528:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077302.769535:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913442816:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077302.769560:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442944:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077302.769583:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077302.769595:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077302/real 1769077302] req@ffff935002a62a00 x1855008913442944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077302.769608:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913442944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077302.769616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077302.769619:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077302.769623:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913442944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077302.769629:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077302.769645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077302.769655:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077302/real 1769077302] req@ffff935002a60000 x1855008913443072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077302.769667:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913443072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077302.769674:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077302.769678:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077302.769681:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077302.769687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077302.769702:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077302.769712:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077302/real 1769077302] req@ffff935002a62300 x1855008913443200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077302.769724:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913443200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077302.769733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077302.769736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077302.769739:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077302.769744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913443328:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077302.769760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077302.769770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077302/real 1769077302] req@ffff935002a61880 x1855008913443328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077302.769781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913443328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077302.769788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077302.769791:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077302.769794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913443328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077302.769800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913443456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077302.769815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077302.769825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077302/real 1769077302] req@ffff935002a61180 x1855008913443456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077302.769836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913443456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077302.769850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077302.769853:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077302.769857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913443456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077302.769863:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913443584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077302.769879:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077302.769888:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077302/real 1769077302] req@ffff935002a60a80 x1855008913443584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077302.769899:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913443584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077302.769907:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077302.769911:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077302.769914:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913443584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077302.961159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077303.473159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077303.473160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077303.473162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077303.473267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077303.793209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077303.985207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077304.497160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077304.497160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077304.497160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077304.497191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077304.817141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077305.009163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077305.521152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077305.521153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077305.521161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077305.521232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077305.841181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077306.033208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077306.545155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077306.545155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077306.545216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077306.545243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077306.865200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077307.057138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077307.569149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077307.569155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077307.569200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077307.569288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077307.889221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077307.889226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.889239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077307.889245:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.889272:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3013 00000100:00080000:0.0:1769077307.889285:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.889293:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077307.889341:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913443712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077307.889371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.889376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077307.889379:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.889386:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3013 00000100:00080000:0.0:1769077307.889391:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.889396:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077307.889405:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913443712:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769077307.889409:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077307.889429:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913443840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077307.889443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.889447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077307.889450:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.889455:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3013 00000100:00080000:0.0:1769077307.889461:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.889466:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077307.889473:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077307.889485:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913443968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077307.889497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077307.889498:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077307.889507:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077307.889510:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.889517:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3013 00000100:00080000:0.0:1769077307.889523:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.889529:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077307.889536:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077307.889548:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff9350103fbb80 x1855008913443712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077307.889557:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913444096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077307.889574:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913443712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077307.889578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077307.889589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077307.889590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077307.889598:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077307.889600:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077307.889612:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913443712:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077307.889617:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3013 00000100:00080000:0.0:1769077307.889629:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.889635:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077307.889642:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077307.889659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443840:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077307.889661:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913444224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077307.889688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.889692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077307.889695:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.889701:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3013 00000800:00000100:2.0:1769077307.889708:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077307.889713:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.889721:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077307.889729:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077307.889733:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff9350103fad80 x1855008913443840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077307.889750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913444352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077307.889759:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913443840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077307.889773:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077307.889784:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077307.889785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.889796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077307.889796:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443840:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077307.889804:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.889816:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3013 00000100:00100000:2.0:1769077307.889817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443968:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077307.889830:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.889842:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077307.889851:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077307.889857:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077307.889869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff9350103fb800 x1855008913443968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077307.889881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913444480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077307.889887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913443968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077307.889967:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077307.889975:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077307.889983:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913443968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077307.889993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077307.890014:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077307.890031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff9350103f9f80 x1855008913444096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077307.890042:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913444096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077307.890073:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077307.890077:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077307.890080:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077307.890096:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913444224:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077307.890109:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077307.890116:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff9350103f9180 x1855008913444224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077307.890129:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913444224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077307.890135:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077307.890138:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077307.890140:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913444224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077307.890144:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913444352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077307.890153:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077307.890159:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff9350103f9500 x1855008913444352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077307.890166:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913444352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077307.890171:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077307.890173:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077307.890175:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913444352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077307.890179:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913444480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077307.890188:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077307.890193:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff9350103fbb80 x1855008913444480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077307.890201:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913444480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077307.890210:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077307.890212:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077307.890215:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913444480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077308.081182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077308.593185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077308.593185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077308.593191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077308.593237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077308.913206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077309.105118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077309.617153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077309.617153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077309.617161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077309.617180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077309.937201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077310.129164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077310.641193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077310.641193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077310.641193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077310.641253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077310.961168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077311.153191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077311.665153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077311.665153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077311.665154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077311.665238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077311.985160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077312.177188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077312.689137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077312.689137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077312.689167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077312.689226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:1.0:1769077313.009167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077313.009172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077313.009177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077313.009181:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077313.009190:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3018 00000100:00080000:1.0:1769077313.009199:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077313.009206:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077313.009238:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913444608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077313.009259:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077313.009262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077313.009263:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077313.009266:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3018 00000100:00080000:1.0:1769077313.009268:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077313.009271:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077313.009278:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077313.009289:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913444736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077313.009296:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913444608:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077313.009298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077313.009304:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077313.009305:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077313.009309:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3018 00000100:00080000:1.0:1769077313.009311:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077313.009313:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077313.009317:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077313.009324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913444864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077313.009331:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077313.009333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077313.009334:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077313.009336:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3018 00000100:00080000:1.0:1769077313.009338:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077313.009340:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077313.009342:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077313.009350:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913444992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077313.009356:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077313.009358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077313.009362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077313.009363:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077313.009367:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3018 00000100:00080000:1.0:1769077313.009370:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077313.009373:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077313.009376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077313.009384:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913445120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077313.009389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077313.009391:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077313/real 1769077313] req@ffff9350100ca680 x1855008913444608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077313.009392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077313.009395:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077313.009401:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3018 00000100:00080000:1.0:1769077313.009407:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077313.009409:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077313.009412:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077313.009414:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913444608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077313.009422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913445248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077313.009424:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077313.009434:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077313.009434:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077313.009439:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077313.009440:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077313.009444:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913444608:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077313.009445:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3018 00000100:00080000:1.0:1769077313.009450:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077313.009453:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077313.009456:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077313.009461:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913445376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077313.009484:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444736:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077313.009504:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077313.009512:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077313/real 1769077313] req@ffff9350100c9f80 x1855008913444736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077313.009521:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913444736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077313.009535:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077313.009538:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077313.009541:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077313.009547:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077313.009566:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077313.009577:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077313/real 1769077313] req@ffff9350100c9c00 x1855008913444864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077313.009586:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913444864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077313.009591:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077313.009594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077313.009596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077313.009599:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077313.009619:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077313.009626:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077313/real 1769077313] req@ffff9350100cb100 x1855008913444992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077313.009637:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913444992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077313.009644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077313.009647:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077313.009651:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913444992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077313.009657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913445120:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077313.009675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077313.009684:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077313/real 1769077313] req@ffff9350100c9180 x1855008913445120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077313.009692:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913445120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077313.009697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077313.009699:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077313.009701:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913445120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077313.009705:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913445248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077313.009716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077313.009721:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077313/real 1769077313] req@ffff9350100c8a80 x1855008913445248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077313.009734:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913445248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077313.009739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077313.009741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077313.009743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913445248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077313.009747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913445376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077313.009757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077313.009763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077313/real 1769077313] req@ffff9350100c8000 x1855008913445376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077313.009769:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913445376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077313.009774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077313.009776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077313.009778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913445376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077313.201133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077313.713140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077313.713141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077313.713162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077313.713172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077314.033275:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077314.225204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077314.737160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077314.737160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077314.737160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077314.737188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077315.057105:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077315.249204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077315.761156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077315.761156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077315.761161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077315.761210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077316.081151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077316.273197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077316.785181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077316.785181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077316.785182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077316.785221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077317.105137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077317.297208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077317.809148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077317.809149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077317.809156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077317.809211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077318.129146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077318.129150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077318.129158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077318.129162:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077318.129171:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3023 00000100:00080000:1.0:1769077318.129182:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077318.129188:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077318.129225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913445504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077318.129242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077318.129245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077318.129246:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077318.129249:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3023 00000100:00080000:1.0:1769077318.129251:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077318.129254:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077318.129263:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077318.129268:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913445504:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077318.129276:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913445632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077318.129284:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077318.129286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077318.129287:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077318.129290:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3023 00000100:00080000:1.0:1769077318.129292:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077318.129293:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077318.129297:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077318.129305:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913445760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077318.129311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077318.129313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077318.129315:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077318.129317:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3023 00000100:00080000:1.0:1769077318.129319:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077318.129321:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077318.129323:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077318.129329:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913445888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077318.129334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077318.129336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077318.129336:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077318.129339:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3023 00000100:00080000:1.0:1769077318.129341:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077318.129344:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077318.129346:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077318.129352:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913446016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077318.129357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077318.129359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077318.129360:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077318.129362:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3023 00000100:00080000:1.0:1769077318.129364:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077318.129366:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077318.129368:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077318.129376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913446144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077318.129381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077318.129383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077318.129384:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077318.129389:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3023 00000100:00100000:2.0:1769077318.129389:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913445632:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077318.129391:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077318.129397:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077318.129401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077318.129409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913446272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077318.129428:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913445760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077318.129454:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913445888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077318.129477:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077318.129496:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077318.129515:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077318.321152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077318.833173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077318.833175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077318.833176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077318.833212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077319.153121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077319.345155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077319.857132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077319.857136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077319.857136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077319.857185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077320.177163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077320.369134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077320.881144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077320.881144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077320.881177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077320.881220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077321.201155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077321.201299:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077321.201330:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077321.201344:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077321.201384:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077321.201394:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077321.201405:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077321.201415:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077321.201422:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077321.201431:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077321.201457:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077318/real 1769077321] req@ffff9350100c9f80 x1855008913446272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077321.201494:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913446272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077321.201507:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077321.201512:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077321.201517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077321.201525:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077318/real 1769077321] req@ffff9350100c8000 x1855008913445632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077321.201539:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913445632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077321.201553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077321.201557:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077321.201560:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913445632:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077321.201566:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077318/real 1769077321] req@ffff9350100c8a80 x1855008913445760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077321.201579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913445760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077321.201586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077321.201590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077321.201592:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913445760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077321.201599:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077318/real 1769077321] req@ffff9350100c9180 x1855008913445888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077321.201612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913445888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077321.201619:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077321.201621:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077321.201624:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913445888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077321.201631:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077318/real 1769077321] req@ffff9350100cb100 x1855008913446016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077321.201641:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913446016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077321.201647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077321.201650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077321.201653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077321.201696:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077318/real 1769077321] req@ffff9350100c9c00 x1855008913446144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077321.201707:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913446144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077321.201744:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077321.201748:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077321.201751:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077321.201762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077318/real 1769077321] req@ffff9350100cad80 x1855008913445504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077321.201775:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913445504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077321.201781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077321.201784:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077321.201786:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913445504:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077321.393147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077321.905133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077321.905150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077321.905174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077321.905222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077322.225159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077322.417162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077322.929144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077322.929144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077322.929255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077322.929299:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077323.249159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077323.249159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077323.249168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077323.249172:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077323.249179:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3028 00000100:00080000:2.0:1769077323.249188:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077323.249194:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077323.249222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913446400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077323.249245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077323.249247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077323.249249:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077323.249251:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3028 00000100:00080000:2.0:1769077323.249253:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077323.249255:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077323.249263:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077323.249275:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913446528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077323.249292:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077323.249294:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913446400:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077323.249297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077323.249302:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077323.249308:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3028 00000100:00080000:2.0:1769077323.249312:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077323.249315:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077323.249321:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077323.249335:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913446656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077323.249345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077323.249348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077323.249350:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077323.249354:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3028 00000100:00080000:2.0:1769077323.249359:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077323.249363:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077323.249368:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769077323.249371:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077323.249385:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913446784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077323.249394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077323.249397:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077323.249399:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077323.249404:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3028 00000100:00100000:0.0:1769077323.249407:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077323/real 1769077323] req@ffff935002a62680 x1855008913446400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077323.249409:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077323.249417:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077323.249425:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077323.249435:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913446400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077323.249441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913446912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077323.249454:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077323.249456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077323.249461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077323.249464:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077323.249465:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077323.249473:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913446400:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077323.249473:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3028 00000100:00080000:2.0:1769077323.249481:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077323.249486:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077323.249491:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077323.249503:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913447040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077323.249504:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913446528:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077323.249519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077323.249522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077323.249524:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077323.249527:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3028 00000100:00080000:2.0:1769077323.249531:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077323.249535:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077323.249536:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077323.249543:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077323.249552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913447168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077323.249556:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077323/real 1769077323] req@ffff935002a61f80 x1855008913446528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077323.249587:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913446528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077323.249595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077323.249598:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077323.249600:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913446528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077323.249605:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913446656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077323.249623:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077323.249633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077323/real 1769077323] req@ffff935002a61c00 x1855008913446656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077323.249645:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913446656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077323.249652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077323.249656:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077323.249659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913446656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077323.249664:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913446784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077323.250432:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077323.250449:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077323/real 1769077323] req@ffff935002a63100 x1855008913446784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077323.250462:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913446784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077323.250470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077323.250474:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077323.250477:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913446784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077323.250483:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446912:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077323.250505:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077323.250516:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077323/real 1769077323] req@ffff935002a60a80 x1855008913446912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077323.250526:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913446912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077323.250533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077323.250537:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077323.250540:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913446912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077323.250567:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077323.250585:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077323.250596:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077323/real 1769077323] req@ffff935002a61180 x1855008913447040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077323.250607:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913447040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077323.250614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077323.250617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077323.250622:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077323.250628:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077323.250651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077323.250661:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077323/real 1769077323] req@ffff935002a61880 x1855008913447168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077323.250672:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913447168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077323.250699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077323.250702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077323.250705:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077323.441142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077323.953143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077323.953163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077323.953166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077323.953183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077324.273160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077324.465149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077324.977148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077324.977149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077324.977150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077324.977214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077325.297162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077325.489164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077326.001116:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077326.001116:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077326.001129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077326.001149:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077326.321152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077326.513125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077327.025129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077327.025129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077327.025132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077327.025201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077327.345149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077327.537140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077328.049185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077328.049186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077328.049187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077328.049188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:2.0:1769077328.369195:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077328.369201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077328.369207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077328.369214:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077328.369226:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3034 00000100:00080000:2.0:1769077328.369238:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077328.369246:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077328.369283:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913447296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077328.369314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077328.369319:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077328.369320:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077328.369326:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3034 00000100:00080000:2.0:1769077328.369330:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077328.369334:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077328.369344:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077328.369358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913447424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077328.369362:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913447296:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077328.369378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077328.369381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077328.369383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077328.369389:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3034 00000100:00080000:2.0:1769077328.369394:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077328.369398:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077328.369403:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077328.369416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913447552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077328.369424:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077328.369429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077328.369434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077328.369436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077328.369440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3034 00000100:00080000:2.0:1769077328.369443:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077328.369447:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077328.369451:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077328.369459:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077328/real 1769077328] req@ffff935002a62300 x1855008913447296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077328.369463:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913447680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077328.369476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077328.369486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077328.369489:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077328.369494:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913447296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077328.369495:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3034 00000100:00080000:2.0:1769077328.369503:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077328.369506:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077328.369509:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077328.369513:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077328.369518:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913447296:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769077328.369518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077328.369534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913447808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077328.369540:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913447424:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077328.369544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077328.369548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077328.369550:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077328.369554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3034 00000100:00080000:2.0:1769077328.369558:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077328.369561:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077328.369564:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077328.369569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077328.369579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913447936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077328.369581:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077328/real 1769077328] req@ffff935002a60000 x1855008913447424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077328.369614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077328.369623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077328.369625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077328.369631:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913447424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077328.369631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3034 00000100:00080000:2.0:1769077328.369639:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077328.369646:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077328.369648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077328.369654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769077328.369657:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077328.369659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913447424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077328.369670:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913447552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077328.369670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913448064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077328.369694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077328.369705:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077328/real 1769077328] req@ffff935002a62a00 x1855008913447552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077328.369712:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913447552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077328.369717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077328.369719:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077328.369721:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913447552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077328.369724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913447680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077328.369734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077328.369739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077328/real 1769077328] req@ffff935002a61500 x1855008913447680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077328.369746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913447680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077328.369751:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077328.369752:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077328.369754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913447680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077328.369765:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447808:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077328.369774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077328.369780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077328/real 1769077328] req@ffff935002a61880 x1855008913447808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077328.369787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913447808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077328.369791:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077328.369793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077328.369795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077328.369798:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077328.369808:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077328.369817:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077328/real 1769077328] req@ffff935002a61180 x1855008913447936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077328.369839:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913447936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077328.369846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077328.369849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077328.369852:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913447936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077328.369857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077328.369873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077328.369883:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077328/real 1769077328] req@ffff935002a60a80 x1855008913448064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077328.369893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913448064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077328.369900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077328.369903:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077328.369906:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077328.561173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077329.073139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077329.073139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077329.073202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077329.073209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077329.393195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077329.585173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077330.097148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077330.097148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077330.097149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077330.097204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077330.417173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077330.609157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077331.121119:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077331.121119:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077331.121144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077331.121147:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077331.441117:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077331.633138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077332.145159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077332.145160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077332.145160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077332.145197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077332.465133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077332.657152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077333.169145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077333.169146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077333.169195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077333.169286:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077333.489133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077333.489149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077333.489156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077333.489161:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077333.489172:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3039 00000100:00080000:2.0:1769077333.489185:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077333.489193:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077333.489234:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913448192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077333.489260:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077333.489262:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077333.489264:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077333.489268:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3039 00000100:00080000:2.0:1769077333.489272:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077333.489276:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077333.489285:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077333.489297:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913448320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077333.489304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077333.489306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077333.489307:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077333.489310:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3039 00000100:00080000:2.0:1769077333.489313:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077333.489316:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077333.489320:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077333.489329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913448448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077333.489340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077333.489341:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913448192:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077333.489351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077333.489354:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077333.489360:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3039 00000100:00080000:2.0:1769077333.489364:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077333.489368:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077333.489375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077333.489389:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913448576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077333.489401:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077333.489404:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077333.489406:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077333.489413:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3039 00000800:00000100:0.0:1769077333.489415:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077333.489418:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077333.489423:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077333.489427:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077333.489439:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913448704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077333.489446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077333.489449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077333.489451:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077333.489454:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3039 00000100:00080000:2.0:1769077333.489457:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077333.489462:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077333.489465:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077333/real 1769077333] req@ffff935002a63100 x1855008913448192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077333.489467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077333.489478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913448832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077333.489499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077333.489530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077333.489532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077333.489537:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3039 00000100:00100000:0.0:1769077333.489542:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913448192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077333.489543:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077333.489551:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077333.489558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077333.489565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769077333.489571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913448960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077333.489575:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077333.489586:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913448192:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077333.489597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913448320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077333.489623:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077333.489635:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077333/real 1769077333] req@ffff935002a61c00 x1855008913448320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077333.489643:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913448320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077333.489650:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077333.489652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077333.489654:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913448320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077333.489672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913448448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077333.489684:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077333.489690:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077333/real 1769077333] req@ffff935002a61f80 x1855008913448448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077333.489697:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913448448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077333.489707:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077333.489709:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077333.489711:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913448448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077333.489715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913448576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077333.489723:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077333.489729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077333/real 1769077333] req@ffff935002a62680 x1855008913448576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077333.489736:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913448576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077333.489741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077333.489743:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077333.489744:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913448576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077333.489748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448704:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077333.489757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077333.489763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077333/real 1769077333] req@ffff935002a60a80 x1855008913448704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077333.489774:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913448704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077333.489783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077333.489786:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077333.489790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077333.489796:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077333.489817:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077333.489830:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077333/real 1769077333] req@ffff935002a61180 x1855008913448832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077333.489843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913448832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077333.489851:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077333.489854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077333.489858:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077333.489865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077333.489883:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077333.489894:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077333/real 1769077333] req@ffff935002a61880 x1855008913448960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077333.489904:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913448960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077333.489910:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077333.489912:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077333.489914:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913448960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077333.681125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077334.193164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077334.193167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077334.193188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077334.193200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077334.513147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077334.705134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077335.217118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077335.217120:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077335.217167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077335.217176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077335.537202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077335.729190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077336.241188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077336.241188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077336.241188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077336.241226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077336.561163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077336.753206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077337.265141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077337.265180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077337.265182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077337.265217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077337.585155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077337.777143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077338.289175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077338.289194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077338.289223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077338.289229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077338.609152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077338.609180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077338.609187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077338.609192:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077338.609202:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3044 00000100:00080000:3.0:1769077338.609223:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077338.609230:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077338.609273:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913449088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077338.609300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077338.609303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077338.609305:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077338.609308:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3044 00000100:00080000:3.0:1769077338.609310:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077338.609313:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077338.609322:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077338.609331:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913449216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077338.609339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077338.609342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077338.609343:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077338.609346:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3044 00000100:00080000:3.0:1769077338.609348:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077338.609353:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609357:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913449088:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077338.609357:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077338.609369:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913449344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077338.609381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077338.609384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077338.609386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077338.609388:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3044 00000100:00080000:3.0:1769077338.609390:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077338.609392:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077338.609395:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077338.609404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913449472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077338.609410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077338.609413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077338.609414:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077338.609436:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3044 00000100:00080000:3.0:1769077338.609442:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077338.609446:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077338.609450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077338.609455:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077338.609465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913449600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077338.609473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077338.609476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077338.609478:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077338.609481:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3044 00000100:00080000:3.0:1769077338.609484:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077338.609487:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077338.609490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077338.609500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913449728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077338.609505:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077338/real 1769077338] req@ffff935002a7f100 x1855008913449088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077338.609510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077338.609518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077338.609522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077338.609528:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3044 00000100:00080000:3.0:1769077338.609530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077338.609533:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913449088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077338.609538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077338.609551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913449856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077338.609553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077338.609564:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077338.609569:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913449088:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077338.609576:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913449216:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077338.609595:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609604:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077338/real 1769077338] req@ffff935002a7c380 x1855008913449216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077338.609613:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913449216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077338.609619:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077338.609621:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077338.609624:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913449216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077338.609661:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913449344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077338.609673:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609679:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077338/real 1769077338] req@ffff935002a7ca80 x1855008913449344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077338.609687:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913449344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077338.609692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077338.609694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077338.609697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913449344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077338.609700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913449472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077338.609710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609715:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077338/real 1769077338] req@ffff935002a7e300 x1855008913449472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077338.609723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913449472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077338.609728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077338.609730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077338.609732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913449472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077338.609735:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913449600:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077338.609744:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609750:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077338/real 1769077338] req@ffff935002a7c700 x1855008913449600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077338.609757:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913449600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077338.609763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077338.609764:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077338.609766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913449600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077338.609769:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913449728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077338.609778:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077338/real 1769077338] req@ffff935002a7f800 x1855008913449728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077338.609791:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913449728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077338.609796:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077338.609811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077338.609814:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913449728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077338.609820:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913449856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077338.609838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077338.609849:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077338/real 1769077338] req@ffff935002a7c000 x1855008913449856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077338.609862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913449856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077338.609871:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077338.609874:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077338.609877:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913449856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077338.801197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077339.313143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077339.313193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077339.313196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077339.313197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077339.633155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077339.825132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077340.337151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077340.337151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077340.337151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077340.337188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077340.657152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077340.849180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077341.361122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077341.361122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077341.361150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077341.361177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077341.681126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077341.873116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077342.385174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077342.385186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077342.385208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077342.385282:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077342.705124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077342.897133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077343.409187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077343.409187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077343.409188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077343.409281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077343.729188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077343.729198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077343.729202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077343.729211:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077343.729223:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3049 00000100:00080000:3.0:1769077343.729236:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077343.729246:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077343.729296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913449984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077343.729359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077343.729365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077343.729368:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077343.729375:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3049 00000100:00080000:3.0:1769077343.729395:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077343.729401:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077343.729411:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913449984:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077343.729415:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077343.729434:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913450112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077343.729448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077343.729452:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077343.729455:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077343.729460:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3049 00000100:00080000:3.0:1769077343.729464:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077343.729469:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077343.729476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077343.729487:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077343.729490:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913450240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077343.729504:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077343.729508:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077343.729511:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077343.729516:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3049 00000100:00080000:3.0:1769077343.729521:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077343.729526:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077343.729532:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077343.729534:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077343/real 1769077343] req@ffff935002a7fb80 x1855008913449984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077343.729548:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913450368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077343.729554:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913449984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077343.729579:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077343.729580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077343.729586:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077343.729588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769077343.729592:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913449984:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077343.729594:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077343.729603:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3049 00000100:00080000:3.0:1769077343.729608:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077343.729613:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077343.729619:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077343.729622:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913450112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077343.729639:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913450496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077343.729651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077343.729657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077343.729662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077343.729664:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077343.729670:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3049 00000100:00100000:0.0:1769077343.729671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077343/real 1769077343] req@ffff935002a7df80 x1855008913450112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077343.729673:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077343.729680:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077343.729688:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077343.729699:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913450112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077343.729702:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913450624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077343.729727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077343.729728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077343.729735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077343.729737:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077343.729738:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077343.729743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913450112:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077343.729745:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3049 00000100:00080000:3.0:1769077343.729752:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077343.729755:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913450240:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077343.729756:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077343.729764:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077343.729775:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913450752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077343.729786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077343.729797:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077343/real 1769077343] req@ffff935002a7d180 x1855008913450240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077343.729807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913450240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077343.729813:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077343.729815:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077343.729817:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913450240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077343.729821:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913450368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077343.729830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077343.729835:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077343/real 1769077343] req@ffff935002a7dc00 x1855008913450368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077343.729842:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913450368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077343.729847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077343.729861:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077343.729867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913450368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077343.729880:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913450496:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077343.729893:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077343.729900:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077343/real 1769077343] req@ffff935002a7fb80 x1855008913450496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077343.729908:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913450496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077343.729914:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077343.729916:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077343.729918:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913450496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077343.729921:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913450624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077343.729931:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077343.729936:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077343/real 1769077343] req@ffff935002a7c000 x1855008913450624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077343.729944:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913450624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077343.729948:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077343.729950:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077343.729952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913450624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077343.729955:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913450752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077343.729964:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077343.729970:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077343/real 1769077343] req@ffff935002a7f800 x1855008913450752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077343.729978:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913450752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077343.729983:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077343.729985:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077343.729987:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913450752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077343.921157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077344.433186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077344.433186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077344.433187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077344.433269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077344.753173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077344.945154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077345.457198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077345.457198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077345.457198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077345.457268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077345.777183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077345.969167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077346.481144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077346.481144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077346.481144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077346.481191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077346.801155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077346.993144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077347.505149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077347.505150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077347.505204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077347.505208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077347.825161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077348.017204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077348.529177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077348.529177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077348.529191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077348.529222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077348.849201:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077348.849213:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077348.849219:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077348.849223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077348.849230:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3054 00000100:00080000:0.0:1769077348.849244:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077348.849278:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077348.849319:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913450880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077348.849352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077348.849355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077348.849357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077348.849360:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3054 00000100:00080000:0.0:1769077348.849362:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077348.849368:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077348.849382:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077348.849400:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913450880:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077348.849401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913451008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077348.849431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077348.849436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077348.849439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077348.849445:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3054 00000100:00080000:0.0:1769077348.849454:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077348.849459:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077348.849466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077348.849476:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077348.849482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913451136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077348.849493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077348.849501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077348.849503:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077348.849511:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3054 00000100:00080000:0.0:1769077348.849518:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077348.849523:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077348/real 1769077348] req@ffff9350103f9c00 x1855008913450880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077348.849526:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077348.849541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077348.849558:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913451264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077348.849567:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913450880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077348.849569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077348.849590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077348.849597:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077348.849601:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077348.849605:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3054 00000100:00080000:0.0:1769077348.849614:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077348.849617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077348.849628:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077348.849628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913450880:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769077348.849649:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077348.849666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077348.849672:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913451392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077348.849688:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077348.849699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077348.849703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077348.849705:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077348.849707:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077348/real 1769077348] req@ffff9350103faa00 x1855008913451008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077348.849713:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3054 00000100:00080000:0.0:1769077348.849727:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077348.849735:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077348.849740:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913451008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077348.849746:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077348.849749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077348.849754:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077348.849758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077348.849764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913451520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077348.849773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451136:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077348.849784:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077348.849787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077348.849789:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077348.849793:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3054 00000100:00080000:0.0:1769077348.849795:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077348.849799:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077348.849805:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077348.849818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913451648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077348.849819:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077348.849848:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077348/real 1769077348] req@ffff9350103fb100 x1855008913451136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077348.849864:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913451136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077348.849875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077348.849886:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077348.849892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077348.849897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077348.849914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077348.849921:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077348/real 1769077348] req@ffff9350103f8700 x1855008913451264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077348.849929:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913451264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077348.849935:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077348.849937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077348.849940:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077348.849954:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913451392:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077348.849965:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077348.849971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077348/real 1769077348] req@ffff9350103f8380 x1855008913451392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077348.849978:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913451392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077348.849984:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077348.849986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077348.849989:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913451392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077348.849993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913451520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077348.850011:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077348.850022:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077348/real 1769077348] req@ffff9350103fb480 x1855008913451520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077348.850035:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913451520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077348.850044:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077348.850047:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077348.850077:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913451520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077348.850082:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913451648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077348.850098:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077348.850106:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077348/real 1769077348] req@ffff9350103f9c00 x1855008913451648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077348.850114:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913451648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077348.850120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077348.850122:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077348.850125:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913451648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077349.041158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077349.553170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077349.553170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077349.553217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077349.553230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077349.873154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077350.065123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077350.577160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077350.577160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077350.577160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077350.577253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077350.897183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077351.089193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077351.601180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077351.601180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077351.601228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077351.601270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077351.921144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077352.113138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077352.625134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077352.625134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077352.625134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077352.625216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077352.945218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077353.137154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077353.649267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077353.649267:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077353.649401:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077353.649430:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077353.969133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077353.969141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.969148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077353.969154:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.969189:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3059 00000100:00080000:3.0:1769077353.969200:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.969207:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077353.969251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913451776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.969276:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.969279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077353.969281:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.969284:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3059 00000100:00080000:3.0:1769077353.969286:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.969289:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077353.969299:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077353.969307:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913451904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.969322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.969327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077353.969328:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.969332:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3059 00000100:00080000:3.0:1769077353.969334:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.969337:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077353.969341:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077353.969348:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913452032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.969361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.969364:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077353.969366:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.969374:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3059 00000100:00100000:2.0:1769077353.969377:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913451776:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077353.969378:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.969391:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077353.969402:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077353.969417:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913452160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.969424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.969427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077353.969428:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.969431:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3059 00000100:00080000:3.0:1769077353.969454:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.969457:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077353.969462:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077353.969470:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077353.969478:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913452288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.969491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.969494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077353.969496:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.969498:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3059 00000100:00080000:3.0:1769077353.969501:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.969506:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077353.969512:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077353.969524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913452416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077353.969526:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff935002a7c700 x1855008913451776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077353.969539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.969552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077353.969557:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.969567:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3059 00000100:00080000:3.0:1769077353.969571:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.969576:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077353.969583:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077353.969587:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913451776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077353.969596:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913452544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077353.969611:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077353.969618:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077353.969623:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913451776:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077353.969632:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451904:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077353.969655:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077353.969664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff935002a7e300 x1855008913451904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077353.969674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913451904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077353.969680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077353.969683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077353.969685:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913451904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077353.969706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077353.969718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077353.969724:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff935002a7ca80 x1855008913452032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077353.969737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913452032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077353.969743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077353.969745:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077353.969747:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077353.969751:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077353.969760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077353.969765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff935002a7c380 x1855008913452160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077353.969773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913452160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077353.969778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077353.969780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077353.969782:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077353.969786:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913452288:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077353.969795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077353.969801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff935002a7f100 x1855008913452288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077353.969808:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913452288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077353.969818:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077353.969821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077353.969823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913452288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077353.969826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913452416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077353.969836:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077353.969841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff935002a7f800 x1855008913452416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077353.969847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913452416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077353.969852:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077353.969854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077353.969856:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913452416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077353.969859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913452544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077353.969868:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077353.969873:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff935002a7c000 x1855008913452544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077353.969880:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913452544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077353.969885:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077353.969888:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077353.969892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913452544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077354.161207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077354.673174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077354.673185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077354.673222:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077354.673256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077354.993182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077355.185150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077355.697163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077355.697169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077355.697181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077355.697200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077356.017169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077356.209205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077356.721133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077356.721152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077356.721156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077356.721223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077357.041226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077357.233126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077357.745163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077357.745163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077357.745171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077357.745231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077358.065182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077358.257172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077358.769145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077358.769146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077358.769147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077358.769182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077359.089252:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077359.089264:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077359.089274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077359.089279:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077359.089291:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3064 00000100:00080000:2.0:1769077359.089306:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077359.089314:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077359.089370:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913452672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077359.089406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077359.089410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077359.089412:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077359.089416:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3064 00000100:00080000:2.0:1769077359.089420:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077359.089424:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077359.089434:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077359.089449:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913452800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077359.089460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077359.089462:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913452672:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077359.089466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077359.089471:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077359.089476:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3064 00000100:00080000:2.0:1769077359.089480:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077359.089485:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077359.089491:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077359.089506:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913452928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077359.089515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077359.089518:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077359.089520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077359.089522:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3064 00000100:00080000:2.0:1769077359.089526:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077359.089532:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077359.089533:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077359.089540:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077359.089550:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913453056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077359.089565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077359.089567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077359.089569:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077359.089572:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3064 00000100:00080000:2.0:1769077359.089576:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077359.089578:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077359.089581:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077359.089590:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913453184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077359.089598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077359.089599:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077359/real 1769077359] req@ffff935002a61500 x1855008913452672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077359.089605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077359.089608:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077359.089617:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3064 00000100:00080000:2.0:1769077359.089622:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077359.089625:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077359.089629:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077359.089631:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913452672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077359.089640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913453312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077359.089641:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077359.089651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077359.089657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077359.089659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913452672:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077359.089665:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077359.089668:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077359.089671:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3064 00000100:00080000:2.0:1769077359.089674:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077359.089676:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077359.089679:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077359.089688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077359.089689:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913453440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077359.089716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077359.089729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077359/real 1769077359] req@ffff935002a62a00 x1855008913452800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077359.089746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913452800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077359.089753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077359.089755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077359.089758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077359.089762:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077359.089774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077359.089779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077359/real 1769077359] req@ffff935002a60000 x1855008913452928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077359.089787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913452928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077359.089792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077359.089794:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077359.089796:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913452928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077359.089799:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077359.089809:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077359.089814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077359/real 1769077359] req@ffff935002a62300 x1855008913453056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077359.089822:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913453056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077359.089828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077359.089830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077359.089832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077359.089835:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913453184:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077359.089845:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077359.089850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077359/real 1769077359] req@ffff935002a61880 x1855008913453184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077359.089856:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913453184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077359.089861:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077359.089863:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077359.089866:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913453184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077359.089869:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913453312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077359.089878:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077359.089883:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077359/real 1769077359] req@ffff935002a61180 x1855008913453312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077359.089890:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913453312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077359.089894:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077359.089896:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077359.089899:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913453312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077359.089910:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913453440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077359.089919:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077359.089925:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077359/real 1769077359] req@ffff935002a60a80 x1855008913453440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077359.089931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913453440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077359.089937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077359.089939:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077359.089941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913453440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077359.281176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077359.793165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077359.793177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077359.793209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077359.793236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077360.113178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077360.305219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077360.817130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077360.817130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077360.817131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077360.817177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077361.137140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077361.329134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077361.841168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077361.841168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077361.841168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077361.841222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077362.161192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077362.353170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077362.865231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077362.865232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077362.865232:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077362.865255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077363.185140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077363.377157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077363.889167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077363.889184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077363.889245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077363.889299:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:0.0:1769077364.209185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077364.209191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077364.209198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077364.209203:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077364.209212:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3069 00000100:00080000:0.0:1769077364.209223:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077364.209229:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077364.209262:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913453568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077364.209287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077364.209292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077364.209293:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077364.209297:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3069 00000100:00080000:0.0:1769077364.209300:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077364.209303:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077364.209312:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077364.209323:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913453696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077364.209326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913453568:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077364.209340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077364.209343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077364.209344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077364.209347:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3069 00000100:00080000:0.0:1769077364.209350:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077364.209352:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077364.209356:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077364.209364:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913453824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077364.209379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077364.209381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077364.209383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077364.209386:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3069 00000100:00080000:0.0:1769077364.209388:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077364.209392:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077364.209396:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077364.209405:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077364.209407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913453952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077364.209415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077364.209418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077364.209419:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077364.209422:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3069 00000100:00080000:0.0:1769077364.209430:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077364.209433:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077364.209436:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077364.209444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913454080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077364.209451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077364.209454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077364.209455:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077364/real 1769077364] req@ffff9350103fbb80 x1855008913453568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077364.209456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077364.209462:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3069 00000100:00080000:0.0:1769077364.209467:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077364.209473:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077364.209479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077364.209495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913454208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077364.209496:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913453568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077364.209513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077364.209522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077364.209524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077364.209526:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077364.209528:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3069 00000100:00080000:0.0:1769077364.209535:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077364.209540:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077364.209541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077364.209547:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077364.209552:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913453568:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077364.209559:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913454336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077364.209582:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453696:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077364.209608:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077364.209622:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077364/real 1769077364] req@ffff9350103f9500 x1855008913453696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077364.209636:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913453696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077364.209645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077364.209649:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077364.209652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453696:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077364.209659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077364.209679:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077364.209692:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077364/real 1769077364] req@ffff9350103f9180 x1855008913453824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077364.209712:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913453824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077364.209721:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077364.209724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077364.209728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077364.209735:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077364.209755:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077364.209767:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077364/real 1769077364] req@ffff9350103f9f80 x1855008913453952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077364.209778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913453952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077364.209787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077364.209791:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077364.209795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913453952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077364.209801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454080:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077364.209822:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077364.209833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077364/real 1769077364] req@ffff9350103fb800 x1855008913454080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077364.209846:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913454080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077364.209861:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077364.209865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077364.209869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077364.209876:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077364.209905:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077364.209918:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077364/real 1769077364] req@ffff9350103fad80 x1855008913454208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077364.209931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913454208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077364.209941:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077364.209944:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077364.209948:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077364.209955:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077364.209974:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077364.210002:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077364/real 1769077364] req@ffff9350103f9c00 x1855008913454336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077364.210019:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913454336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077364.210028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077364.210031:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077364.210036:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077364.401200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077364.913154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077364.913201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077364.913208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077364.913209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077365.233173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077365.425166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077365.937175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077365.937175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077365.937181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077365.937225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077366.257216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077366.449182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077366.961183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077366.961191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077366.961220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077366.961283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077367.281146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077367.473135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077367.985172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077367.985172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077367.985172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077367.985213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077368.305261:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077368.497194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077369.009182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077369.009191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077369.009208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077369.009283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077369.329187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077369.329198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077369.329202:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077369.329212:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3075 00000100:00100000:2.0:1769077369.329216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077369.329223:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077369.329232:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077369.329267:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913454464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077369.329292:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077369.329296:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077369.329297:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077369.329300:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3075 00000100:00080000:1.0:1769077369.329302:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077369.329305:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077369.329314:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077369.329326:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913454592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077369.329336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077369.329338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077369.329339:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077369.329346:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3075 00000100:00100000:2.0:1769077369.329346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913454464:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077369.329355:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077369.329364:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077369.329369:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077369.329382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913454720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077369.329390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077369.329392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077369.329393:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077369.329395:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3075 00000100:00080000:1.0:1769077369.329398:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077369.329401:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077369.329404:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077369.329415:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913454848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077369.329423:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077369.329426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077369.329427:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077369.329430:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3075 00000100:00080000:1.0:1769077369.329433:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077369.329436:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077369.329440:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077369.329446:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913454976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077369.329453:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077369.329465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077369.329469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077369.329470:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077369.329473:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3075 00000100:00080000:1.0:1769077369.329479:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077369.329483:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077369.329488:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077369.329500:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913455104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077369.329513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077369.329515:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077369/real 1769077369] req@ffff9350100ca680 x1855008913454464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077369.329517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077369.329521:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077369.329537:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3075 00000100:00080000:1.0:1769077369.329548:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077369.329552:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077369.329559:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077369.329563:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913454464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077369.329573:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913455232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077369.329577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077369.329586:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077369.329591:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913454464:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077369.329622:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913454592:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077369.329639:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077369.329648:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077369/real 1769077369] req@ffff9350100c8e00 x1855008913454592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077369.329656:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913454592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077369.329663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077369.329665:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077369.329668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913454592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077369.329672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913454720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077369.329685:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077369.329691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077369/real 1769077369] req@ffff9350100c9880 x1855008913454720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077369.329699:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913454720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077369.329704:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077369.329706:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077369.329708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913454720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077369.329713:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913454848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077369.329722:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077369.329727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077369/real 1769077369] req@ffff9350100c9500 x1855008913454848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077369.329734:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913454848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077369.329739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077369.329741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077369.329743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913454848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077369.329747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454976:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077369.329756:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077369.329762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077369/real 1769077369] req@ffff9350100caa00 x1855008913454976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077369.329780:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913454976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077369.329789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077369.329792:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077369.329796:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913454976:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077369.329803:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913455104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077369.329824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077369.329834:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077369/real 1769077369] req@ffff9350100c8380 x1855008913455104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077369.329842:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913455104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077369.329847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077369.329849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077369.329852:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913455104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077369.329856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913455232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077369.329867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077369.329873:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077369/real 1769077369] req@ffff9350100ca300 x1855008913455232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077369.329888:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913455232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077369.329898:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077369.329914:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077369.329917:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913455232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077369.521202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077370.033128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077370.033139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077370.033189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077370.033192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077370.353266:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077370.545151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077371.057168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077371.057175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077371.057203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077371.057232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077371.377169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077371.569201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077372.081127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077372.081127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077372.081145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077372.081166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077372.401177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077372.593179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077373.105126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077373.105127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077373.105141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077373.105204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077373.425125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077373.617166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077374.129151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077374.129151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077374.129166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077374.129224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077374.449225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077374.449228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077374.449238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077374.449242:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077374.449252:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3080 00000100:00080000:3.0:1769077374.449263:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077374.449270:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077374.449313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913455360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077374.449340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077374.449343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077374.449345:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077374.449348:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3080 00000100:00080000:3.0:1769077374.449350:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077374.449353:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077374.449362:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077374.449371:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913455488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077374.449379:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077374.449382:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913455360:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077374.449383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077374.449387:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077374.449392:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3080 00000100:00080000:3.0:1769077374.449394:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077374.449397:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077374.449400:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077374.449409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913455616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077374.449415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077374.449417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077374.449419:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077374.449422:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3080 00000100:00080000:3.0:1769077374.449425:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077374.449427:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077374.449430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077374.449437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913455744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077374.449443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077374.449445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077374.449446:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077374.449449:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3080 00000100:00080000:3.0:1769077374.449452:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077374.449457:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077374.449457:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077374.449465:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077374.449474:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913455872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077374.449481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077374.449484:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077374.449485:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077374.449488:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3080 00000100:00080000:3.0:1769077374.449490:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077374.449492:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077374.449495:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077374.449504:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913456000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077374.449510:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077374/real 1769077374] req@ffff935002a7fb80 x1855008913455360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077374.449527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077374.449534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077374.449535:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077374.449538:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3080 00000100:00080000:3.0:1769077374.449542:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077374.449544:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077374.449547:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077374.449549:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913455360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077374.449556:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913456128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077374.449562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077374.449569:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077374.449572:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913455360:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077374.449598:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913455488:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077374.449617:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077374.449626:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077374/real 1769077374] req@ffff935002a7dc00 x1855008913455488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077374.449639:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913455488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077374.449645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077374.449648:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077374.449650:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913455488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077374.449655:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913455616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077374.449674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077374.449686:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077374/real 1769077374] req@ffff935002a7d180 x1855008913455616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077374.449699:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913455616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077374.449708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077374.449712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077374.449715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913455616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077374.449721:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913455744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077374.449739:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077374.449747:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077374/real 1769077374] req@ffff935002a7df80 x1855008913455744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077374.449755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913455744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077374.449765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077374.449767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077374.449769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913455744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077374.449773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913455872:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077374.449784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077374.449789:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077374/real 1769077374] req@ffff935002a7c000 x1855008913455872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077374.449796:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913455872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077374.449802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077374.449804:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077374.449805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913455872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077374.449809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077374.449839:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077374.449847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077374/real 1769077374] req@ffff935002a7f800 x1855008913456000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077374.449855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913456000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077374.449860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077374.449862:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077374.449865:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077374.449869:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077374.449880:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077374.449885:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077374/real 1769077374] req@ffff935002a7f100 x1855008913456128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077374.449892:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913456128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077374.449897:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077374.449898:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077374.449900:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077374.641131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077375.153187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077375.153187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077375.153195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077375.153242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077375.473229:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077375.665204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077376.177130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077376.177160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077376.177161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077376.177255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077376.497160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077376.689150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077377.201186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077377.201186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077377.201186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077377.201224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077377.521178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077377.713122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077378.225146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077378.225149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077378.225207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077378.225211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077378.545220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077378.737159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077379.249171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077379.249171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077379.249171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077379.249237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077379.569167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077379.569178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077379.569183:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077379.569195:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3085 00000100:00100000:2.0:1769077379.569196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077379.569210:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077379.569221:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077379.569261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913456256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077379.569287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077379.569291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077379.569292:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077379.569296:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3085 00000100:00080000:1.0:1769077379.569299:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077379.569302:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077379.569312:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077379.569324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913456384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077379.569333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077379.569337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077379.569341:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077379.569341:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913456256:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077379.569353:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3085 00000100:00080000:1.0:1769077379.569358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077379.569360:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077379.569364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077379.569377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913456512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077379.569388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077379.569391:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077379.569392:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077379.569395:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3085 00000100:00080000:1.0:1769077379.569399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077379.569402:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077379.569405:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077379.569414:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913456640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077379.569420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077379.569424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077379.569426:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077379.569430:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3085 00000100:00080000:1.0:1769077379.569434:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077379.569439:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077379.569444:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077379.569454:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077379.569472:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913456768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077379.569485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077379.569489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077379.569491:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077379.569496:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077379/real 1769077379] req@ffff9350100cb800 x1855008913456256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077379.569503:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3085 00000100:00080000:1.0:1769077379.569518:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077379.569523:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913456256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077379.569526:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077379.569544:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077379.569546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077379.569554:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077379.569559:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913456256:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077379.569566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913456896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077379.569578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077379.569582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077379.569585:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077379.569586:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913456384:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077379.569599:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3085 00000100:00080000:1.0:1769077379.569607:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077379.569607:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077379.569620:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077379.569625:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077379/real 1769077379] req@ffff9350100ca300 x1855008913456384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077379.569629:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077379.569645:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913456384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077379.569658:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769077379.569663:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913457024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077379.569664:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077379.569675:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913456384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077379.569691:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913456512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077379.569710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077379.569720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077379/real 1769077379] req@ffff9350100c8380 x1855008913456512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077379.569744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913456512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077379.569751:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077379.569754:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077379.569756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913456512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077379.569765:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913456640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077379.569786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077379.569807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077379/real 1769077379] req@ffff9350100caa00 x1855008913456640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077379.569822:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913456640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077379.569830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077379.569834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077379.569837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913456640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077379.569841:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456768:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077379.569858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077379.569865:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077379/real 1769077379] req@ffff9350100c9500 x1855008913456768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077379.569873:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913456768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077379.569879:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077379.569881:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077379.569883:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077379.569896:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077379.569907:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077379.569913:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077379/real 1769077379] req@ffff9350100c9880 x1855008913456896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077379.569920:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913456896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077379.569930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077379.569934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077379.569937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913456896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077379.569959:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077379.569996:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077379.570014:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077379/real 1769077379] req@ffff9350100c8e00 x1855008913457024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077379.570028:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913457024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077379.570033:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077379.570036:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077379.570038:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077379.761154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077380.273165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077380.273165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077380.273165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077380.273207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077380.593207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077380.785123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077381.297177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077381.297177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077381.297177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077381.297205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077381.617150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077381.809199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077382.321184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077382.321188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077382.321215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077382.321270:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077382.641130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077382.833186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077383.345193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077383.345193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077383.345211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077383.345243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077383.665153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077383.857205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077384.369221:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077384.369224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077384.369228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077384.369292:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077384.689167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077384.689176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077384.689178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077384.689184:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077384.689210:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3090 00000100:00080000:0.0:1769077384.689223:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077384.689231:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077384.689274:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913457152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077384.689303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077384.689306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077384.689308:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077384.689311:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3090 00000100:00080000:0.0:1769077384.689315:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077384.689318:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077384.689325:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077384.689338:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913457280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077384.689346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077384.689349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077384.689350:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077384.689357:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3090 00000100:00080000:0.0:1769077384.689359:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077384.689362:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077384.689365:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077384.689377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913457408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077384.689387:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913457152:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077384.689410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077384.689415:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077384.689418:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077384.689425:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3090 00000100:00080000:0.0:1769077384.689431:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077384.689435:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077384.689441:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077384.689459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913457536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077384.689470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077384.689475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077384.689477:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077384.689480:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3090 00000100:00080000:0.0:1769077384.689484:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077384.689487:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077384.689492:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077384.689502:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913457664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077384.689509:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077384.689512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077384.689514:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077384.689516:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3090 00000100:00080000:0.0:1769077384.689521:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077384.689523:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913457280:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077384.689528:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077384.689534:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077384.689550:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913457792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077384.689559:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077384.689562:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077384.689564:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077384.689566:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3090 00000100:00080000:0.0:1769077384.689579:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077384.689581:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077384.689587:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077384.689589:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913457408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077384.689615:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913457920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077384.689621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913457536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077384.689691:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077384.689712:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077384.689729:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077384.881155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077385.393145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077385.393146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077385.393215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077385.393247:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077385.713155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077385.905174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077386.417194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077386.417194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077386.417207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077386.417275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077386.737142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077386.929158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077387.441162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077387.441167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077387.441168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077387.441221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077387.761154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077387.761311:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077387.761346:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077387.761362:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077387.761396:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077387.761405:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077387.761413:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077387.761421:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077387.761428:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077387.761437:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077387.761524:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077384/real 1769077387] req@ffff9350103fad80 x1855008913457920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077387.761567:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913457920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077387.761582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077387.761589:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077387.761610:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077387.761620:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077384/real 1769077387] req@ffff9350103f8700 x1855008913457408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077387.761635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913457408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077387.761644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077387.761647:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077387.761651:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913457408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077387.761659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077384/real 1769077387] req@ffff9350103fb100 x1855008913457536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077387.761674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913457536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077387.761683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077387.761688:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077387.761692:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913457536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077387.761701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077384/real 1769077387] req@ffff9350103faa00 x1855008913457664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077387.761713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913457664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077387.761721:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077387.761724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077387.761728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077387.761736:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077384/real 1769077387] req@ffff9350103f9c00 x1855008913457792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077387.761748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913457792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077387.761756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077387.761760:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077387.761763:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913457792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077387.761771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077384/real 1769077387] req@ffff9350103fb480 x1855008913457152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077387.761787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913457152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077387.761794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077387.761798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077387.761802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913457152:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077387.761813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077384/real 1769077387] req@ffff9350103f8380 x1855008913457280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077387.761832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913457280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077387.761841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077387.761845:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077387.761848:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913457280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077387.953177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077388.465192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077388.465192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077388.465207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077388.465250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077388.785146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077388.977224:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077389.489160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077389.489162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077389.489162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077389.489212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077389.809173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077389.809173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077389.809184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077389.809189:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077389.809198:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3095 00000100:00080000:0.0:1769077389.809209:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077389.809216:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077389.809252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913458048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077389.809278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077389.809282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077389.809283:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077389.809292:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3095 00000100:00080000:0.0:1769077389.809297:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077389.809301:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077389.809311:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077389.809322:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913458176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077389.809330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077389.809335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077389.809336:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913458048:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077389.809338:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077389.809347:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3095 00000100:00080000:0.0:1769077389.809352:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077389.809356:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077389.809362:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077389.809376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913458304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077389.809385:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077389.809389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077389.809391:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077389.809397:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3095 00000100:00080000:0.0:1769077389.809401:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077389.809407:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077389.809413:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077389.809425:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913458432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077389.809434:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077389.809438:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077389.809440:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077389.809445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3095 00000100:00080000:0.0:1769077389.809449:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077389.809453:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077389.809458:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077389.809466:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077389.809471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913458560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077389.809482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077389.809486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077389.809489:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077389.809493:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3095 00000100:00080000:0.0:1769077389.809498:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077389.809502:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077389.809508:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077389.809523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913458688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077389.809529:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077389/real 1769077389] req@ffff9350103fb800 x1855008913458048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077389.809538:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077389.809548:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077389.809551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077389.809556:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3095 00000100:00080000:0.0:1769077389.809576:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077389.809583:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077389.809583:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913458048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077389.809591:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077389.809604:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077389.809611:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913458816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077389.809613:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077389.809624:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913458048:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077389.809667:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913458176:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077389.809695:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077389.809710:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077389/real 1769077389] req@ffff9350103f9f80 x1855008913458176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077389.809724:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913458176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077389.809734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077389.809738:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077389.809741:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913458176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077389.809748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913458304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077389.809772:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077389.809791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077389/real 1769077389] req@ffff9350103f9180 x1855008913458304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077389.809802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913458304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077389.809810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077389.809814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077389.809817:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913458304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077389.809823:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913458432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077389.809843:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077389.809857:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077389/real 1769077389] req@ffff9350103f9500 x1855008913458432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077389.809870:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913458432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077389.809884:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077389.809888:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077389.809891:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913458432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077389.809897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913458560:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077389.809918:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077389.809930:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077389/real 1769077389] req@ffff9350103fbb80 x1855008913458560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077389.809950:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913458560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077389.809959:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077389.809962:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077389.809965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913458560:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077389.809972:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913458688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077389.809992:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077389.810003:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077389/real 1769077389] req@ffff9350103f8380 x1855008913458688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077389.810016:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913458688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077389.810025:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077389.810029:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077389.810032:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913458688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077389.810046:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913458816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077389.810111:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077389.810124:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077389/real 1769077389] req@ffff9350103fb480 x1855008913458816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077389.810136:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913458816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077389.810145:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077389.810149:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077389.810152:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913458816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077390.001164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077390.513188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077390.513189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077390.513189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077390.513226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077390.833158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077391.025202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077391.537109:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077391.537116:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077391.537130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077391.537170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077391.857213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077392.049134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077392.561182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077392.561182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077392.561182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077392.561210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077392.881131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077393.073159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077393.585151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077393.585152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077393.585152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077393.585225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077393.905157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077394.097139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077394.609181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077394.609205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077394.609218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077394.609260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077394.929216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077394.929266:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077394.929276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077394.929281:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077394.929291:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3100 00000100:00080000:0.0:1769077394.929303:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077394.929310:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077394.929353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913458944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077394.929380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077394.929383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077394.929386:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077394.929389:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3100 00000100:00080000:0.0:1769077394.929392:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077394.929396:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077394.929407:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077394.929414:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913458944:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077394.929419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913459072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077394.929429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077394.929431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077394.929433:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077394.929436:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3100 00000100:00080000:0.0:1769077394.929439:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077394.929441:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077394.929445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077394.929461:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913459200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077394.929468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077394.929482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077394.929484:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077394.929488:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3100 00000100:00080000:0.0:1769077394.929492:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077394.929494:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077394.929499:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077394.929510:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913459328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077394.929510:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077394.929519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077394.929522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077394.929523:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077394.929526:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3100 00000100:00080000:0.0:1769077394.929529:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077394.929532:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077394.929535:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077394.929542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913459456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077394.929548:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077394.929551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077394.929552:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077394.929555:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3100 00000100:00080000:0.0:1769077394.929558:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077394.929561:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9350103f9c00 x1855008913458944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077394.929563:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077394.929568:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077394.929580:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913459584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077394.929586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077394.929589:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913458944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077394.929590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077394.929594:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077394.929602:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3100 00000100:00080000:0.0:1769077394.929607:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077394.929612:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077394.929619:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077394.929623:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077394.929628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913459712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077394.929638:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077394.929643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913458944:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077394.929668:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459072:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077394.929686:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077394.929695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9350103faa00 x1855008913459072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077394.929704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913459072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077394.929710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077394.929712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077394.929715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459072:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077394.929719:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077394.929730:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077394.929736:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9350103fb100 x1855008913459200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077394.929743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913459200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077394.929749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077394.929750:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077394.929752:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077394.929757:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077394.929766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077394.929779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9350103f8700 x1855008913459328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077394.929786:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913459328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077394.929792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077394.929794:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077394.929796:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077394.929800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913459456:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077394.929811:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077394.929816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9350103fad80 x1855008913459456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077394.929826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913459456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077394.929835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077394.929838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077394.929849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913459456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077394.929856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913459584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077394.929877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077394.929890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9350103fb480 x1855008913459584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077394.929903:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913459584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077394.929912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077394.929915:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077394.929918:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913459584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077394.929923:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913459712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077394.929938:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077394.929946:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9350103f8380 x1855008913459712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077394.929954:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913459712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077394.929959:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077394.929961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077394.929963:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913459712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077395.121151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077395.633164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077395.633170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077395.633172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077395.633220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077395.953149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077396.145125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077396.657160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077396.657162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077396.657192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077396.657200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077396.977134:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077397.169150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077397.681127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077397.681131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077397.681136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077397.681251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077398.001157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077398.193119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077398.705131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077398.705141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077398.705147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077398.705182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077399.025170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077399.217147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077399.729109:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077399.729113:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077399.729118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077399.729154:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077400.049139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077400.049152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077400.049167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077400.049171:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077400.049181:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3105 00000100:00080000:2.0:1769077400.049192:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077400.049198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077400.049236:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913459840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077400.049259:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077400.049261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077400.049266:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077400.049270:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3105 00000100:00080000:2.0:1769077400.049273:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077400.049276:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077400.049286:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077400.049295:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913459968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077400.049305:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913459840:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077400.049305:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077400.049310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077400.049313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077400.049319:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3105 00000100:00080000:2.0:1769077400.049321:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077400.049324:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077400.049327:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077400.049337:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913460096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077400.049343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077400.049346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077400.049348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077400.049350:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3105 00000100:00080000:2.0:1769077400.049353:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077400.049356:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077400.049358:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077400.049365:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913460224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077400.049370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077400.049372:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077400.049373:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077400.049376:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3105 00000100:00080000:2.0:1769077400.049381:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077400.049398:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077400.049403:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077400.049408:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077400.049421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913460352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077400.049430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077400.049432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077400.049434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077400.049436:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3105 00000100:00080000:2.0:1769077400.049440:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077400.049443:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077400.049448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077400.049456:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077400/real 1769077400] req@ffff935002a62680 x1855008913459840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077400.049458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913460480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077400.049477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077400.049485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077400.049487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077400.049489:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3105 00000100:00080000:2.0:1769077400.049492:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077400.049495:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077400.049498:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077400.049500:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913459840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077400.049507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913460608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077400.049525:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077400.049533:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077400.049538:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913459840:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077400.049566:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459968:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077400.049591:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077400.049606:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077400/real 1769077400] req@ffff935002a61f80 x1855008913459968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077400.049639:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913459968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077400.049649:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077400.049654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077400.049657:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913459968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077400.049665:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077400.049689:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077400.049702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077400/real 1769077400] req@ffff935002a61c00 x1855008913460096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077400.049716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913460096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077400.049725:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077400.049729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077400.049732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077400.049739:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077400.049759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077400.049771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077400/real 1769077400] req@ffff935002a63100 x1855008913460224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077400.049784:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913460224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077400.049800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077400.049803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077400.049807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077400.049813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913460352:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077400.049832:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077400.049844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077400/real 1769077400] req@ffff935002a60a80 x1855008913460352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077400.049857:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913460352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077400.049866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077400.049869:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077400.049873:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913460352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077400.049879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913460480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077400.049899:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077400.049910:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077400/real 1769077400] req@ffff935002a61180 x1855008913460480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077400.049922:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913460480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077400.049931:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077400.049934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077400.049938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913460480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077400.049944:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913460608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077400.049961:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077400.049982:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077400/real 1769077400] req@ffff935002a61880 x1855008913460608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077400.049995:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913460608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077400.050004:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077400.050008:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077400.050011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913460608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077400.241116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077400.753119:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077400.753170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077400.753172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077400.753252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077401.073182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077401.265176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077401.777164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077401.777164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077401.777164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077401.777208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077402.097135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077402.289117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077402.801164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077402.801164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077402.801164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077402.801196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077403.121127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077403.313191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077403.825150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077403.825150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077403.825157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077403.825185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077404.145123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077404.337113:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077404.849150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077404.849150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077404.849151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077404.849182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077405.169155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077405.169164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077405.169169:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077405.169169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077405.169179:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3110 00000100:00080000:1.0:1769077405.169190:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077405.169197:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077405.169232:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913460736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077405.169264:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077405.169267:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077405.169268:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077405.169272:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3110 00000100:00080000:1.0:1769077405.169274:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077405.169277:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077405.169285:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077405.169294:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913460864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077405.169317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077405.169320:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077405.169322:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077405.169326:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3110 00000100:00080000:1.0:1769077405.169329:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077405.169332:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077405.169336:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077405.169338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913460736:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077405.169347:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913460992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077405.169355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077405.169357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077405.169358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077405.169360:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3110 00000100:00080000:1.0:1769077405.169362:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077405.169365:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077405.169368:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077405.169375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913461120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077405.169382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077405.169384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077405.169385:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077405.169388:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3110 00000100:00080000:1.0:1769077405.169398:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077405.169401:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077405.169405:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077405.169412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913461248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077405.169419:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077405.169424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077405.169428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077405.169429:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077405.169432:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3110 00000100:00080000:1.0:1769077405.169434:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077405.169437:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077405.169441:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077405.169448:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913461376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077405.169455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077405.169458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077405.169460:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077405.169463:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3110 00000100:00100000:0.0:1769077405.169472:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077405/real 1769077405] req@ffff9350100ca680 x1855008913460736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077405.169472:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077405.169481:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077405.169488:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077405.169499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913461504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077405.169506:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913460736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077405.169516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077405.169521:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077405.169525:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913460736:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077405.169531:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077405.169551:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077405.169561:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077405/real 1769077405] req@ffff9350100cad80 x1855008913460864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077405.169569:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913460864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077405.169575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077405.169578:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077405.169580:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077405.169599:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077405.169610:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077405.169616:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077405/real 1769077405] req@ffff9350100c9c00 x1855008913460992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077405.169623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913460992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077405.169629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077405.169631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077405.169633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913460992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077405.169638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913461120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077405.169654:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077405.169665:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077405/real 1769077405] req@ffff9350100cb100 x1855008913461120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077405.169678:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913461120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077405.169687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077405.169691:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077405.169694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913461120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077405.169701:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913461248:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077405.169722:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077405.169733:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077405/real 1769077405] req@ffff9350100c9180 x1855008913461248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077405.169743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913461248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077405.169752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077405.169755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077405.169758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913461248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077405.169764:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913461376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077405.169784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077405.169795:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077405/real 1769077405] req@ffff9350100c8a80 x1855008913461376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077405.169806:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913461376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077405.169812:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077405.169814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077405.169816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913461376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077405.169820:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913461504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077405.169832:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077405.169839:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077405/real 1769077405] req@ffff9350100c8000 x1855008913461504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077405.169846:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913461504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077405.169853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077405.169857:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077405.169860:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913461504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077405.361170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077405.873142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077405.873142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077405.873142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077405.873185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077406.193158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077406.385178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077406.897167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077406.897168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077406.897192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077406.897208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077407.217159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077407.409132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077407.921150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077407.921151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077407.921173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077407.921189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077408.241136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077408.433109:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077408.945166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077408.945166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077408.945173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077408.945231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077409.265146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077409.457184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077409.969206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077409.969206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077409.969206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077409.969258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077410.289111:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077410.289153:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077410.289161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077410.289165:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077410.289172:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3115 00000100:00080000:1.0:1769077410.289183:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077410.289189:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077410.289235:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913461632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077410.289257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077410.289260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077410.289261:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077410.289277:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3115 00000100:00080000:1.0:1769077410.289280:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077410.289284:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077410.289292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077410.289298:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913461632:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077410.289308:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913461760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077410.289316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077410.289319:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077410.289320:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077410.289323:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3115 00000100:00080000:1.0:1769077410.289326:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077410.289328:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077410.289332:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077410.289341:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913461888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077410.289358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077410.289360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077410.289361:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077410.289364:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3115 00000100:00080000:1.0:1769077410.289366:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077410.289369:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077410.289371:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077410.289378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913462016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077410.289385:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077410.289391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077410.289395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077410.289396:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077410.289400:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3115 00000100:00080000:1.0:1769077410.289404:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077410.289406:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077410.289410:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077410.289419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913462144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077410.289425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077410.289430:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077410/real 1769077410] req@ffff9350100c9f80 x1855008913461632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077410.289430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077410.289434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077410.289443:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3115 00000100:00080000:1.0:1769077410.289447:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077410.289450:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077410.289454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077410.289455:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913461632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077410.289464:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769077410.289465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913462272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077410.289470:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077410.289478:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913461632:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077410.289479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077410.289484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077410.289486:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077410.289489:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3115 00000100:00080000:1.0:1769077410.289492:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077410.289494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077410.289497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077410.289503:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913461760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077410.289506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913462400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077410.289528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077410.289539:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077410/real 1769077410] req@ffff9350100c8000 x1855008913461760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077410.289547:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913461760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077410.289553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077410.289556:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077410.289558:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913461760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077410.289562:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913461888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077410.289581:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077410.289592:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077410/real 1769077410] req@ffff9350100c8a80 x1855008913461888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077410.289606:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913461888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077410.289615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077410.289618:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077410.289622:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913461888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077410.289628:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077410.289644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077410.289651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077410/real 1769077410] req@ffff9350100c9180 x1855008913462016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077410.289659:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913462016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077410.289665:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077410.289667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077410.289669:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077410.289672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913462144:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077410.289682:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077410.289687:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077410/real 1769077410] req@ffff9350100cb100 x1855008913462144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077410.289695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913462144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077410.289705:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077410.289707:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077410.289710:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913462144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077410.289713:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913462272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077410.289722:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077410.289727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077410/real 1769077410] req@ffff9350100c9c00 x1855008913462272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077410.289734:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913462272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077410.289739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077410.289740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077410.289742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913462272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077410.289754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913462400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077410.289764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077410.289770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077410/real 1769077410] req@ffff9350100cad80 x1855008913462400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077410.289777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913462400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077410.289782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077410.289784:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077410.289787:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913462400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077410.481154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077410.993159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077410.993159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077410.993224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077410.993225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077411.313164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077411.505134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077412.017158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077412.017158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077412.017163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077412.017224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077412.337127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077412.529171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077413.041134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077413.041134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077413.041134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077413.041214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077413.361194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077413.553141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077414.065199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077414.065201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077414.065216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077414.065236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077414.385139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077414.577196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077415.089202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077415.089202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077415.089251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077415.089253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077415.409178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077415.409196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077415.409204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077415.409209:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077415.409219:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3121 00000100:00080000:3.0:1769077415.409232:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077415.409239:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077415.409275:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913462528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077415.409300:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077415.409306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077415.409307:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077415.409310:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3121 00000100:00080000:3.0:1769077415.409313:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077415.409316:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077415.409323:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077415.409333:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913462656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077415.409341:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077415.409344:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077415.409346:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077415.409348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913462528:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077415.409351:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3121 00000100:00080000:3.0:1769077415.409358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077415.409360:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077415.409364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077415.409374:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913462784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077415.409380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077415.409383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077415.409384:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077415.409387:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3121 00000100:00080000:3.0:1769077415.409390:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077415.409392:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077415.409396:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077415.409402:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913462912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077415.409408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077415.409410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077415.409411:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077415.409414:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3121 00000100:00080000:3.0:1769077415.409417:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077415.409420:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077415.409424:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077415.409428:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077415.409438:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913463040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077415.409445:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077415.409447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077415.409448:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077415.409451:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3121 00000100:00080000:3.0:1769077415.409453:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077415.409456:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077415.409459:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077415.409462:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077415/real 1769077415] req@ffff935002a7c380 x1855008913462528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077415.409470:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913463168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077415.409479:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077415.409482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077415.409483:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077415.409488:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913462528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077415.409488:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3121 00000100:00080000:3.0:1769077415.409493:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077415.409500:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077415.409508:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077415.409511:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077415.409519:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077415.409523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913463296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077415.409528:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913462528:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077415.409562:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462656:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077415.409582:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077415.409591:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077415/real 1769077415] req@ffff935002a7ca80 x1855008913462656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077415.409600:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913462656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077415.409607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077415.409609:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077415.409611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077415.409615:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077415.409627:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077415.409632:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077415/real 1769077415] req@ffff935002a7e300 x1855008913462784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077415.409639:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913462784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077415.409645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077415.409647:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077415.409648:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077415.409652:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077415.409661:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077415.409667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077415/real 1769077415] req@ffff935002a7c700 x1855008913462912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077415.409674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913462912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077415.409679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077415.409681:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077415.409683:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913462912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077415.409687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463040:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077415.409696:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077415.409702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077415/real 1769077415] req@ffff935002a7f100 x1855008913463040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077415.409709:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913463040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077415.409714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077415.409716:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077415.409718:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077415.409721:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077415.409730:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077415.409735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077415/real 1769077415] req@ffff935002a7f800 x1855008913463168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077415.409742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913463168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077415.409747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077415.409749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077415.409751:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077415.409754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077415.409762:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077415.409769:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077415/real 1769077415] req@ffff935002a7c000 x1855008913463296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077415.409776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913463296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077415.409781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077415.409783:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077415.409785:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077415.601118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077416.113168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077416.113195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077416.113200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077416.113206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077416.433179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077416.625181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077417.137176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077417.137177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077417.137181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077417.137224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077417.457142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077417.649120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077418.161152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077418.161153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077418.161261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077418.161298:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077418.481189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077418.673155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077419.185146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077419.185150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077419.185165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077419.185179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077419.505156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077419.697132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077420.209147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077420.209147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077420.209154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077420.209195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077420.529141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077420.529159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077420.529165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077420.529170:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077420.529178:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3126 00000100:00080000:2.0:1769077420.529187:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077420.529193:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077420.529228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913463424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077420.529251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077420.529254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077420.529256:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077420.529260:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3126 00000100:00080000:2.0:1769077420.529262:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077420.529265:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077420.529273:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077420.529284:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913463424:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077420.529291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913463552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077420.529300:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077420.529303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077420.529304:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077420.529307:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3126 00000100:00080000:2.0:1769077420.529310:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077420.529313:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077420.529316:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077420.529325:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913463680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077420.529332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077420.529335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077420.529337:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077420.529339:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3126 00000100:00080000:2.0:1769077420.529342:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077420.529345:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077420.529350:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769077420.529352:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077420.529363:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913463808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077420.529371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077420.529373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077420.529375:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077420.529378:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3126 00000100:00080000:2.0:1769077420.529381:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077420.529384:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077420.529387:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077420.529389:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077420/real 1769077420] req@ffff935002a62300 x1855008913463424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077420.529398:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913463936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077420.529405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077420.529408:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913463424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077420.529409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077420.529413:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077420.529421:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3126 00000100:00080000:2.0:1769077420.529426:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077420.529427:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077420.529431:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077420.529436:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769077420.529436:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077420.529443:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913463424:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077420.529448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913464064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077420.529460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077420.529463:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077420.529466:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913463552:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077420.529466:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077420.529473:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3126 00000100:00080000:2.0:1769077420.529476:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077420.529478:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077420.529482:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077420.529494:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077420.529495:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913464192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077420.529510:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077420/real 1769077420] req@ffff935002a60000 x1855008913463552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077420.529528:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913463552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077420.529535:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077420.529537:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077420.529540:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913463552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077420.529545:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913463680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077420.529557:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077420.529564:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077420/real 1769077420] req@ffff935002a62a00 x1855008913463680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077420.529571:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913463680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077420.529577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077420.529579:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077420.529581:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913463680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077420.529584:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913463808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077420.529594:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077420.529600:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077420/real 1769077420] req@ffff935002a61500 x1855008913463808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077420.529608:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913463808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077420.529613:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077420.529615:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077420.529617:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913463808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077420.529620:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077420.529629:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077420.529641:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077420/real 1769077420] req@ffff935002a61880 x1855008913463936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077420.529649:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913463936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077420.529655:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077420.529657:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077420.529659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913463936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077420.529664:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077420.529682:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077420.529694:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077420/real 1769077420] req@ffff935002a61180 x1855008913464064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077420.529715:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913464064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077420.529724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077420.529735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077420.529740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077420.529760:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077420.529776:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077420.529784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077420/real 1769077420] req@ffff935002a60a80 x1855008913464192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077420.529792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913464192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077420.529797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077420.529800:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077420.529802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077420.721151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077421.233141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077421.233142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077421.233147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077421.233179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077421.553147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077421.745148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077422.257151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077422.257151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077422.257151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077422.257188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077422.577154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077422.769179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077423.281155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077423.281155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077423.281186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077423.281192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077423.601138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077423.793124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077424.305142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077424.305145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077424.305255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077424.305292:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077424.625170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077424.817155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077425.329167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077425.329167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077425.329167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077425.329334:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077425.649122:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077425.649132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077425.649136:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077425.649144:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3131 00000100:00080000:2.0:1769077425.649154:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077425.649160:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077425.649193:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913464320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077425.649216:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077425.649219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077425.649221:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077425.649224:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3131 00000100:00080000:2.0:1769077425.649226:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077425.649229:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077425.649237:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077425.649246:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913464448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077425.649253:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077425.649255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077425.649256:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077425.649259:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3131 00000100:00080000:2.0:1769077425.649261:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077425.649263:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077425.649265:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077425.649273:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913464576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077425.649278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077425.649280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077425.649281:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077425.649283:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3131 00000100:00080000:2.0:1769077425.649285:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077425.649288:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077425.649291:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077425.649297:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913464704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077425.649302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077425.649305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077425.649307:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077425.649311:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3131 00000100:00080000:2.0:1769077425.649314:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077425.649318:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077425.649323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077425.649336:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913464832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077425.649346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077425.649349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077425.649351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077425.649355:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3131 00000100:00080000:2.0:1769077425.649359:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077425.649363:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077425.649368:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077425.649377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913464960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077425.649383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077425.649386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077425.649387:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077425.649390:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3131 00000100:00080000:2.0:1769077425.649392:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077425.649395:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077425.649397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077425.649404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913465088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077425.649502:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913464320:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769077425.649580:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077425.649614:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077425/real 1769077425] req@ffff935002a63100 x1855008913464320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077425.649627:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913464320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077425.649637:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077425.649641:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077425.649646:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913464320:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077425.649653:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913464448:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077425.649678:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077425.649691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077425/real 1769077425] req@ffff935002a61c00 x1855008913464448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077425.649703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913464448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077425.649710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077425.649713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077425.649715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913464448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077425.649720:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913464576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077425.649742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077425.649750:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077425/real 1769077425] req@ffff935002a61f80 x1855008913464576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077425.649759:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913464576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077425.649765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077425.649767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077425.649769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913464576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077425.649772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913464704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077425.649782:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077425.649788:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077425/real 1769077425] req@ffff935002a62680 x1855008913464704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077425.649795:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913464704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077425.649800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077425.649802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077425.649804:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913464704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077425.649807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077425.649817:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077425.649822:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077425/real 1769077425] req@ffff935002a60a80 x1855008913464832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077425.649830:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913464832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077425.649840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077425.649842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077425.649845:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077425.649848:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077425.649858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077425.649868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077425/real 1769077425] req@ffff935002a61180 x1855008913464960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077425.649879:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913464960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077425.649887:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077425.649890:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077425.649894:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913464960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077425.649901:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077425.649920:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077425.649929:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077425/real 1769077425] req@ffff935002a61880 x1855008913465088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077425.649937:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913465088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077425.649943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077425.649945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077425.649947:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077425.650034:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077425.841112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077426.353136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077426.353137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077426.353138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077426.353201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077426.673196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077426.865211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077427.377213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077427.377214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077427.377214:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077427.377294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077427.697137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077427.889129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077428.401178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077428.401183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077428.401191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077428.401232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077428.721158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077428.913197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077429.425200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077429.425204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077429.425237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077429.425276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077429.745139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077429.937183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077430.449170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077430.449201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077430.449202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077430.449254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077430.769184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077430.769196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077430.769205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077430.769209:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077430.769219:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3136 00000100:00080000:3.0:1769077430.769232:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077430.769239:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077430.769278:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913465216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077430.769313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077430.769317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077430.769319:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077430.769323:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3136 00000100:00080000:3.0:1769077430.769326:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077430.769330:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077430.769339:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077430.769352:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913465344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077430.769364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077430.769369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077430.769373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077430.769378:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913465216:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077430.769380:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3136 00000100:00080000:3.0:1769077430.769387:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077430.769397:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077430.769402:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077430.769417:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913465472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077430.769427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077430.769430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077430.769433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077430.769437:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3136 00000100:00080000:3.0:1769077430.769447:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077430.769451:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077430.769457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077430.769472:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913465600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077430.769477:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077430.769496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077430.769501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077430.769505:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077430.769512:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3136 00000100:00080000:3.0:1769077430.769517:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077430.769523:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077430.769530:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077430.769537:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077430/real 1769077430] req@ffff935002a7df80 x1855008913465216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077430.769552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913465728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077430.769557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913465216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077430.769617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077430.769620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077430.769624:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077430.769630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077430.769632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913465216:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077430.769637:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077430.769647:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3136 00000100:00080000:3.0:1769077430.769652:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077430.769657:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077430.769663:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913465344:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077430.769664:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077430.769690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913465856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077430.769691:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077430.769710:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077430.769712:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077430/real 1769077430] req@ffff935002a7d180 x1855008913465344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077430.769731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077430.769736:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077430.769741:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913465344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077430.769749:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3136 00000100:00080000:0.0:1769077430.769769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077430.769770:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077430.769777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077430.769784:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913465344:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077430.769785:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077430.769793:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913465472:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769077430.769796:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077430.769817:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913465984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077430.769818:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077430.769830:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077430/real 1769077430] req@ffff935002a7dc00 x1855008913465472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077430.769847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913465472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077430.769853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077430.769856:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077430.769858:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913465472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077430.769864:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913465600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077430.769899:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077430.769907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077430/real 1769077430] req@ffff935002a7fb80 x1855008913465600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077430.769915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913465600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077430.769921:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077430.769923:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077430.769926:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913465600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077430.769930:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077430.769943:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077430.769949:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077430/real 1769077430] req@ffff935002a7c000 x1855008913465728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077430.769957:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913465728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077430.769962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077430.769964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077430.769966:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077430.769985:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077430.769996:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077430.770003:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077430/real 1769077430] req@ffff935002a7f800 x1855008913465856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077430.770011:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913465856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077430.770016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077430.770018:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077430.770020:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077430.770024:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077430.770034:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077430.770040:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077430/real 1769077430] req@ffff935002a7f100 x1855008913465984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077430.770081:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913465984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077430.770087:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077430.770089:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077430.770092:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913465984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077430.961201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077431.473163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077431.473163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077431.473163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077431.473203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077431.793162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077431.985179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077432.497128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077432.497128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077432.497129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077432.497183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077432.817175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077433.009159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077433.521160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077433.521161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077433.521162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077433.521186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077433.841126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077434.033149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077434.545117:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077434.545118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077434.545119:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077434.545156:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077434.865170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077435.057158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077435.569160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077435.569166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077435.569166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077435.569200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077435.889189:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077435.889192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077435.889205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077435.889211:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.889225:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3141 00000100:00080000:3.0:1769077435.889240:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.889247:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077435.889294:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913466112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077435.889329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.889334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077435.889336:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.889341:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3141 00000100:00080000:3.0:1769077435.889345:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.889349:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077435.889360:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077435.889373:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913466240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077435.889381:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913466112:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077435.889390:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.889393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077435.889396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.889400:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3141 00000100:00080000:3.0:1769077435.889402:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.889405:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077435.889412:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077435.889420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913466368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077435.889427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.889430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077435.889432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.889434:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3141 00000100:00080000:3.0:1769077435.889436:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.889438:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077435.889442:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077435.889452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913466496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077435.889464:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077435.889474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.889477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077435.889479:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.889483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3141 00000100:00080000:3.0:1769077435.889486:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.889489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077435.889494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077435.889508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913466624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077435.889515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.889518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077435.889519:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.889523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3141 00000100:00080000:3.0:1769077435.889528:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077435.889529:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff935002a7c700 x1855008913466112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077435.889535:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077435.889550:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077435.889565:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913466112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077435.889570:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913466752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077435.889609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.889613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077435.889614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077435.889615:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.889625:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3141 00000100:00080000:3.0:1769077435.889630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077435.889631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077435.889638:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077435.889645:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913466112:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077435.889646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077435.889662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913466880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077435.889687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913466240:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077435.889724:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077435.889741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff935002a7e300 x1855008913466240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077435.889756:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913466240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077435.889767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077435.889771:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077435.889775:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913466240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077435.889796:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913466368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077435.889820:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077435.889833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff935002a7ca80 x1855008913466368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077435.889843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913466368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077435.889849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077435.889852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077435.889854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913466368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077435.889859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913466496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077435.889873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077435.889880:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff935002a7c380 x1855008913466496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077435.889888:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913466496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077435.889894:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077435.889896:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077435.889899:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913466496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077435.889906:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913466624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077435.889925:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077435.889937:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff935002a7f100 x1855008913466624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077435.889950:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913466624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077435.889959:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077435.889963:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077435.889967:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913466624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077435.889972:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913466752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077435.889988:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077435.889995:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff935002a7f800 x1855008913466752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077435.890003:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913466752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077435.890012:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077435.890015:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077435.890017:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913466752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077435.890033:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913466880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077435.890043:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077435.890071:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff935002a7c000 x1855008913466880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077435.890079:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913466880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077435.890084:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077435.890086:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077435.890089:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913466880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077436.081183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077436.593125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077436.593138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077436.593140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077436.593187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077436.913148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077437.105146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077437.617139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077437.617148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077437.617162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077437.617249:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077437.937171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077438.129106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077438.641148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077438.641157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077438.641166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077438.641178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077438.961144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077439.153136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077439.665148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077439.665150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077439.665188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077439.665194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077439.985192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077440.177193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077440.689140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077440.689161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077440.689176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077440.689188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077441.009191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077441.009210:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077441.009219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077441.009223:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077441.009234:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3146 00000100:00080000:1.0:1769077441.009244:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077441.009253:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077441.009288:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913467008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077441.009316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077441.009319:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077441.009327:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077441.009330:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3146 00000100:00080000:1.0:1769077441.009333:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077441.009335:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077441.009343:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077441.009353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913467136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077441.009361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077441.009363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077441.009365:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077441.009369:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913467008:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077441.009370:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3146 00000100:00080000:1.0:1769077441.009373:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077441.009375:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077441.009379:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077441.009393:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913467264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077441.009404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077441.009407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077441.009410:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077441.009415:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3146 00000100:00080000:1.0:1769077441.009428:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077441.009433:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077441.009439:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077441.009440:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077441.009456:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913467392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077441.009468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077441.009472:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077441.009475:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077441.009480:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077441/real 1769077441] req@ffff9350100ca680 x1855008913467008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077441.009484:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3146 00000100:00080000:1.0:1769077441.009493:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077441.009499:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077441.009503:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913467008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077441.009507:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077441.009514:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077441.009518:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077441.009522:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913467008:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077441.009526:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913467520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077441.009537:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077441.009539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077441.009541:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077441.009545:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913467136:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077441.009546:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3146 00000100:00080000:1.0:1769077441.009553:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077441.009557:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077441.009562:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077441.009571:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077441.009576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913467648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077441.009583:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077441/real 1769077441] req@ffff9350100c8e00 x1855008913467136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077441.009586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077441.009593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077441.009602:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913467136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077441.009604:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077441.009613:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3146 00000100:00080000:0.0:1769077441.009614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077441.009617:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077441.009619:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077441.009623:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077441.009624:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913467136:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769077441.009630:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077441.009633:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913467264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077441.009643:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913467776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077441.009651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077441.009661:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077441/real 1769077441] req@ffff9350100c9880 x1855008913467264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077441.009682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913467264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077441.009688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077441.009690:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077441.009693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913467264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077441.009707:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913467392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077441.009721:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077441.009728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077441/real 1769077441] req@ffff9350100c9500 x1855008913467392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077441.009736:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913467392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077441.009742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077441.009744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077441.009746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913467392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077441.009750:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913467520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077441.009760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077441.009765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077441/real 1769077441] req@ffff9350100caa00 x1855008913467520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077441.009773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913467520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077441.009783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077441.009785:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077441.009787:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913467520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077441.009798:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913467648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077441.009807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077441.009813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077441/real 1769077441] req@ffff9350100c8380 x1855008913467648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077441.009819:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913467648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077441.009825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077441.009826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077441.009828:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913467648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077441.009832:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913467776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077441.009847:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077441.009854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077441/real 1769077441] req@ffff9350100ca300 x1855008913467776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077441.009861:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913467776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077441.009867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077441.009868:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077441.009871:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913467776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077441.201117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077441.713120:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077441.713123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077441.713132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077441.713170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077442.033157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077442.225157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077442.737119:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077442.737127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077442.737129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077442.737182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077443.057111:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077443.249122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077443.761132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077443.761142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077443.761156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077443.761166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077444.081164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077444.273136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077444.785140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077444.785141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077444.785194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077444.785195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077445.105131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077445.297138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077445.809123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077445.809126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077445.809139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077445.809168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077446.129153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077446.129161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077446.129165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077446.129174:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077446.129187:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3151 00000100:00080000:1.0:1769077446.129200:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077446.129208:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077446.129251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913467904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077446.129278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077446.129282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077446.129283:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077446.129287:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3151 00000100:00080000:1.0:1769077446.129289:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077446.129292:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077446.129303:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077446.129320:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913467904:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077446.129320:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913468032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077446.129342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077446.129345:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077446.129346:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077446.129352:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3151 00000100:00080000:1.0:1769077446.129356:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077446.129359:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077446.129363:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077446.129379:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913468160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077446.129387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077446.129389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077446.129390:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077446.129393:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3151 00000100:00080000:1.0:1769077446.129397:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077446.129400:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077446.129405:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077446.129413:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077446.129416:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913468288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077446.129425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077446.129428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077446.129430:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077446.129433:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3151 00000100:00080000:1.0:1769077446.129445:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077446.129451:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077446.129455:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077446.129461:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077446/real 1769077446] req@ffff9350100cb800 x1855008913467904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077446.129465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913468416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077446.129526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077446.129529:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077446.129531:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077446.129537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913467904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077446.129540:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3151 00000100:00080000:1.0:1769077446.129553:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077446.129557:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077446.129562:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077446.129568:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077446.129578:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913467904:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077446.129578:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077446.129622:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913468544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077446.129631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468032:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077446.129639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077446.129642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077446.129645:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077446.129650:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3151 00000100:00080000:1.0:1769077446.129654:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077446.129659:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077446.129665:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077446.129668:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077446.129690:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077446/real 1769077446] req@ffff9350100ca300 x1855008913468032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077446.129690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913468672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077446.129744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913468032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077446.129764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077446.129767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077446.129773:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077446.129779:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077446.129793:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077446.129799:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077446/real 1769077446] req@ffff9350100c8380 x1855008913468160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077446.129807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913468160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077446.129812:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077446.129814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077446.129817:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077446.129820:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077446.129830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077446.129836:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077446/real 1769077446] req@ffff9350100caa00 x1855008913468288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077446.129842:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913468288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077446.129848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077446.129858:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077446.129861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077446.129865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913468416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077446.129879:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077446.129887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077446/real 1769077446] req@ffff9350100c9500 x1855008913468416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077446.129896:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913468416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077446.129903:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077446.129905:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077446.129908:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913468416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077446.129922:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913468544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077446.129934:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077446.129940:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077446/real 1769077446] req@ffff9350100c9880 x1855008913468544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077446.129948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913468544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077446.129953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077446.129955:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077446.129957:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913468544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077446.129961:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913468672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077446.129970:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077446.129977:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077446/real 1769077446] req@ffff9350100c8e00 x1855008913468672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077446.129984:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913468672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077446.129989:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077446.129991:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077446.129993:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913468672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077446.321130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077446.833147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077446.833147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077446.833147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077446.833188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077447.153114:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077447.345125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077447.857132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077447.857150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077447.857162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077447.857223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077448.177169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077448.369143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077448.881151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077448.881151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077448.881151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077448.881177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077449.201162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077449.393149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077449.905130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077449.905130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077449.905150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077449.905183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077450.225128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077450.417123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077450.929152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077450.929164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077450.929164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077450.929181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077451.249136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077451.249176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077451.249182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077451.249186:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077451.249194:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3156 00000100:00080000:3.0:1769077451.249204:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077451.249210:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077451.249243:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913468800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077451.249265:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077451.249268:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077451.249269:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077451.249272:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3156 00000100:00080000:3.0:1769077451.249274:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077451.249277:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077451.249285:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077451.249296:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913468800:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077451.249297:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913468928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077451.249308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077451.249311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077451.249312:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077451.249315:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3156 00000100:00080000:3.0:1769077451.249318:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077451.249320:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077451.249324:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077451.249333:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913469056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077451.249343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077451.249345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077451.249347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077451.249350:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3156 00000100:00080000:3.0:1769077451.249352:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077451.249354:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077451.249358:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077451.249365:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913469184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077451.249371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077451.249374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077451.249375:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077451.249378:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3156 00000100:00080000:3.0:1769077451.249381:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077451.249385:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077451.249389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077451.249399:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077451.249401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913469312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077451.249412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077451.249415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077451.249416:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077451.249419:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3156 00000100:00080000:3.0:1769077451.249431:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077451.249434:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469056:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077451.249438:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077451.249445:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077451.249457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913469440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077451.249461:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469184:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077451.249468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077451.249472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077451.249473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077451.249477:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3156 00000100:00080000:3.0:1769077451.249481:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077451.249484:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077451.249488:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913469312:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077451.249488:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077451.249505:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913469568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077451.249517:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913469440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077451.249567:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913469568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077451.441149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077451.953168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077451.953177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077451.953178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077451.953240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077452.273170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077452.465159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077452.977150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077452.977150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077452.977171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077452.977184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077453.297115:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077453.489140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077454.001125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077454.001132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077454.001159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077454.001172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077454.321173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:1.0:1769077454.321325:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769077454.321361:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769077454.321379:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769077454.321424:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769077454.321435:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769077454.321445:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769077454.321456:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769077454.321467:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769077454.321495:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077454.321508:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077451/real 1769077454] req@ffff935002a7c000 x1855008913469312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077454.321537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913469312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077454.321551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077454.321557:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077454.321562:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913469312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077454.321569:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077451/real 1769077454] req@ffff935002a7f800 x1855008913469440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077454.321579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913469440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077454.321585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077454.321587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077454.321591:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913469440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077454.321598:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077451/real 1769077454] req@ffff935002a7dc00 x1855008913468928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077454.321626:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913468928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077454.321637:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077454.321641:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077454.321645:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913468928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077454.321652:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077451/real 1769077454] req@ffff935002a7d180 x1855008913469056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077454.321662:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913469056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077454.321668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077454.321670:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077454.321673:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077454.321680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077451/real 1769077454] req@ffff935002a7df80 x1855008913469184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077454.321699:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913469184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077454.321722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077454.321726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077454.321731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077454.321740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077451/real 1769077454] req@ffff935002a7fb80 x1855008913468800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077454.321754:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913468800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077454.321762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077454.321766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077454.321770:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913468800:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077454.321806:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077451/real 1769077454] req@ffff935002a7f100 x1855008913469568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077454.321818:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913469568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077454.321825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077454.321828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077454.321831:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913469568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077454.513130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077455.025154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077455.025154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077455.025160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077455.025185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077455.345144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077455.537145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077456.049153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077456.049154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077456.049156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077456.049187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077456.369132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077456.369196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077456.369203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077456.369208:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077456.369217:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3162 00000100:00080000:1.0:1769077456.369227:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077456.369234:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077456.369269:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913469696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077456.369299:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077456.369303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077456.369305:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077456.369309:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3162 00000100:00080000:1.0:1769077456.369313:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077456.369317:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077456.369327:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077456.369340:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913469824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077456.369347:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913469696:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077456.369358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077456.369363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077456.369365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077456.369370:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3162 00000100:00080000:1.0:1769077456.369374:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077456.369378:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077456.369384:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077456.369398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913469952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077456.369414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077456.369418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077456.369419:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077456.369425:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3162 00000800:00000100:0.0:1769077456.369432:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077456.369433:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077456.369455:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077456.369460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077456.369486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913470080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077456.369497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077456.369501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077456.369503:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077456.369509:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3162 00000100:00080000:1.0:1769077456.369513:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077456.369515:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077456/real 1769077456] req@ffff9350100ca680 x1855008913469696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077456.369520:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077456.369529:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077456.369543:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913469696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077456.369550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913470208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077456.369579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077456.369582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077456.369584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077456.369590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077456.369594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077456.369599:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3162 00000100:00100000:0.0:1769077456.369605:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913469696:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077456.369607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077456.369615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077456.369621:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077456.369634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913470336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077456.369643:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469824:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077456.369647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077456.369652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077456.369655:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077456.369661:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3162 00000100:00080000:1.0:1769077456.369673:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077456.369675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077456.369682:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077456.369690:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077456.369695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077456/real 1769077456] req@ffff9350100cad80 x1855008913469824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077456.369708:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913470464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077456.369715:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913469824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077456.369732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077456.369737:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077456.369741:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077456.369748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077456.369771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077456.369782:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077456/real 1769077456] req@ffff9350100c9c00 x1855008913469952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077456.369793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913469952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077456.369802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077456.369805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077456.369809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913469952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077456.369815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077456.369834:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077456.369845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077456/real 1769077456] req@ffff9350100cb100 x1855008913470080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077456.369855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913470080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077456.369862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077456.369865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077456.369868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077456.369874:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913470208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077456.369890:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077456.369900:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077456/real 1769077456] req@ffff9350100c9180 x1855008913470208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077456.369911:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913470208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077456.369919:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077456.369922:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077456.369925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913470208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077456.369943:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913470336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077456.369963:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077456.369975:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077456/real 1769077456] req@ffff9350100c8a80 x1855008913470336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077456.369989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913470336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077456.369997:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077456.370002:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077456.370006:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913470336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077456.370013:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913470464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077456.370042:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077456.370095:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077456/real 1769077456] req@ffff9350100c8000 x1855008913470464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077456.370108:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913470464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077456.370115:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077456.370119:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077456.370122:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913470464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077456.561121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077457.073133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077457.073143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077457.073146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077457.073185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077457.393161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077457.585167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077458.097144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077458.097156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077458.097170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077458.097172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077458.417110:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077458.609114:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077459.121161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077459.121161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077459.121161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077459.121194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077459.441157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077459.633126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077460.145138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077460.145138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077460.145141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077460.145177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077460.465126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077460.657125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077461.169132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077461.169132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077461.169164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077461.169165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077461.489180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077461.489184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077461.489210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077461.489216:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077461.489228:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3167 00000100:00080000:2.0:1769077461.489244:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077461.489254:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077461.489316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913470592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077461.489389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077461.489393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077461.489395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077461.489399:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3167 00000100:00080000:2.0:1769077461.489402:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077461.489406:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077461.489415:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077461.489432:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913470592:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077461.489433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913470720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077461.489450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077461.489453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077461.489455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077461.489459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3167 00000100:00080000:2.0:1769077461.489462:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077461.489465:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077461.489469:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077461.489482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913470848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077461.489489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077461.489492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077461.489493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077461.489496:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3167 00000100:00080000:2.0:1769077461.489507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077461.489510:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077461.489515:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769077461.489515:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077461.489529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913470976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077461.489539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077461.489541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077461.489543:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077461.489546:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3167 00000100:00080000:2.0:1769077461.489549:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077461.489552:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077461.489557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077461.489562:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077461/real 1769077461] req@ffff935002a61500 x1855008913470592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077461.489572:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913471104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077461.489582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077461.489587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077461.489588:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913470592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077461.489590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077461.489601:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3167 00000100:00080000:2.0:1769077461.489607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077461.489611:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077461.489612:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769077461.489616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077461.489621:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077461.489627:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913470592:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077461.489630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913471232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077461.489639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077461.489642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077461.489644:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077461.489646:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3167 00000100:00080000:2.0:1769077461.489650:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077461.489658:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470720:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077461.489658:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077461.489666:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077461.489680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913471360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077461.489693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077461.489709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077461/real 1769077461] req@ffff935002a62a00 x1855008913470720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077461.489720:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913470720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077461.489728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077461.489730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077461.489733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077461.489738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077461.489750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077461.489757:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077461/real 1769077461] req@ffff935002a60000 x1855008913470848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077461.489767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913470848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077461.489774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077461.489776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077461.489778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077461.489782:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077461.489795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077461.489801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077461/real 1769077461] req@ffff935002a62300 x1855008913470976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077461.489810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913470976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077461.489817:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077461.489819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077461.489821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913470976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077461.489825:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913471104:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077461.489836:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077461.489843:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077461/real 1769077461] req@ffff935002a61880 x1855008913471104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077461.489851:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913471104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077461.489863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077461.489866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077461.489868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913471104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077461.489872:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913471232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077461.489883:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077461.489890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077461/real 1769077461] req@ffff935002a61180 x1855008913471232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077461.489898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913471232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077461.489905:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077461.489907:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077461.489909:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913471232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077461.489924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913471360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077461.489936:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077461.489943:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077461/real 1769077461] req@ffff935002a60a80 x1855008913471360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077461.489952:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913471360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077461.489958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077461.489960:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077461.489963:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913471360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077461.681159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077462.193121:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077462.193122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077462.193166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077462.193194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077462.513131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077462.705167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077463.217150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077463.217155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077463.217181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077463.217198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077463.537170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077463.729157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077464.241171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077464.241178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077464.241187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077464.241217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077464.561112:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077464.753110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077465.265136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077465.265137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077465.265137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077465.265170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077465.585142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077465.777120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077466.289151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077466.289151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077466.289153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077466.289198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077466.609148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077466.609164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077466.609170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077466.609174:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077466.609182:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3172 00000100:00080000:3.0:1769077466.609193:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077466.609200:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077466.609250:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913471488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077466.609306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077466.609309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077466.609311:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077466.609314:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3172 00000100:00080000:3.0:1769077466.609318:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077466.609321:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077466.609330:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077466.609344:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913471488:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077466.609344:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913471616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077466.609358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077466.609360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077466.609362:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077466.609365:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3172 00000100:00080000:3.0:1769077466.609368:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077466.609370:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077466.609374:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077466.609383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913471744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077466.609389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077466.609392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077466.609394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077466.609397:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3172 00000100:00080000:3.0:1769077466.609401:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077466.609404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077466.609408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077466.609412:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077466.609416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913471872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077466.609425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077466.609427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077466.609429:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077466.609432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3172 00000100:00080000:3.0:1769077466.609434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077466.609436:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077466.609440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077466.609447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913472000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077466.609453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077466.609455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077466.609457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077466.609459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3172 00000100:00080000:3.0:1769077466.609462:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077466.609464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077466.609468:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077466.609473:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077466/real 1769077466] req@ffff935002a7c380 x1855008913471488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077466.609478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913472128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077466.609492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077466.609495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077466.609496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077466.609501:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913471488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077466.609501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3172 00000100:00080000:3.0:1769077466.609506:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077466.609511:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077466.609516:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077466.609525:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913472256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077466.609527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077466.609539:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077466.609545:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913471488:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077466.609572:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913471616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077466.609593:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077466.609603:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077466/real 1769077466] req@ffff935002a7ca80 x1855008913471616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077466.609618:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913471616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077466.609625:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077466.609628:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077466.609630:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913471616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077466.609634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913471744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077466.609647:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077466.609653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077466/real 1769077466] req@ffff935002a7e300 x1855008913471744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077466.609661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913471744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077466.609666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077466.609668:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077466.609670:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913471744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077466.609674:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913471872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077466.609684:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077466.609689:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077466/real 1769077466] req@ffff935002a7c700 x1855008913471872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077466.609696:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913471872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077466.609701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077466.609703:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077466.609706:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913471872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077466.609710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472000:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077466.609720:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077466.609725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077466/real 1769077466] req@ffff935002a7f100 x1855008913472000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077466.609732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913472000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077466.609738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077466.609740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077466.609742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077466.609745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077466.609754:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077466.609759:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077466/real 1769077466] req@ffff935002a7fb80 x1855008913472128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077466.609766:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913472128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077466.609772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077466.609778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077466.609780:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077466.609784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077466.609794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077466.609801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077466/real 1769077466] req@ffff935002a7df80 x1855008913472256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077466.609809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913472256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077466.609814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077466.609816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077466.609818:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077466.801181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077467.313138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077467.313138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077467.313199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077467.313201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077467.633155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077467.825153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077468.337172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077468.337177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077468.337205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077468.337218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077468.657149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077468.849201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077469.361145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077469.361180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077469.361201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077469.361206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077469.681156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077469.873147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077470.385219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077470.385221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077470.385229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077470.385336:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077470.705144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077470.897171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077471.409121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077471.409138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077471.409160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077471.409228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077471.729175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077471.729221:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077471.729227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077471.729231:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077471.729241:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3177 00000100:00080000:0.0:1769077471.729252:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077471.729259:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077471.729296:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913472384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077471.729328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077471.729332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077471.729335:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077471.729340:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3177 00000100:00080000:0.0:1769077471.729344:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077471.729349:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077471.729359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077471.729368:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913472384:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077471.729375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913472512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077471.729389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077471.729392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077471.729395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077471.729399:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3177 00000100:00080000:0.0:1769077471.729403:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077471.729408:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077471.729413:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077471.729425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913472640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077471.729433:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077471.729441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077471.729446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077471.729449:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077471.729454:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3177 00000100:00080000:0.0:1769077471.729458:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077471.729463:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077471.729469:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077471.729471:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077471/real 1769077471] req@ffff9350103fbb80 x1855008913472384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077471.729484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913472768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077471.729489:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913472384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077471.729498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077471.729503:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077471.729510:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913472384:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077471.729515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077471.729520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077471.729523:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077471.729528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3177 00000100:00080000:0.0:1769077471.729532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077471.729536:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077471.729541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077471.729547:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913472512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077471.729555:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913472896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077471.729567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077471.729571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077471.729573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077471.729580:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3177 00000800:00000100:2.0:1769077471.729581:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077471.729585:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077471.729590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077471.729595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077471.729603:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077471/real 1769077471] req@ffff9350103f9500 x1855008913472512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077471.729611:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913473024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077471.729630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077471.729634:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913472512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077471.729635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077471.729639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077471.729648:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3177 00000100:00080000:2.0:1769077471.729648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077471.729654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077471.729655:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077471.729664:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077471.729664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913472512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077471.729672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913472640:192.168.204.121@tcp:8:kworker.0 00000008:00100000:0.0:1769077471.729673:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077471.729690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913473152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077471.729699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077471.729714:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077471/real 1769077471] req@ffff9350103f9180 x1855008913472640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077471.729723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913472640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077471.729729:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077471.729732:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077471.729735:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913472640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077471.729739:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913472768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077471.729749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077471.729755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077471/real 1769077471] req@ffff9350103f9f80 x1855008913472768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077471.729763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913472768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077471.729768:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077471.729770:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077471.729772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913472768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077471.729792:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472896:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077471.729804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077471.729810:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077471/real 1769077471] req@ffff9350103fb800 x1855008913472896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077471.729818:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913472896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077471.729824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077471.729826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077471.729828:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913472896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077471.729831:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077471.729839:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077471.729845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077471/real 1769077471] req@ffff9350103fbb80 x1855008913473024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077471.729852:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913473024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077471.729858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077471.729860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077471.729862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077471.729865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077471.729873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077471.729878:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077471/real 1769077471] req@ffff9350103f8380 x1855008913473152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077471.729885:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913473152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077471.729889:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077471.729891:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077471.729893:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077471.921120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077472.433139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077472.433144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077472.433211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077472.433229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077472.753180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077472.945152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077473.457132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077473.457133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077473.457140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077473.457191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077473.777224:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077473.969180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077474.481204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077474.481205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077474.481215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077474.481288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077474.801235:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077474.993204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077475.505206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077475.505207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077475.505221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077475.505256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077475.825198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077476.017164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077476.529207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077476.529207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077476.529379:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077476.529420:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:0.0:1769077476.849227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.849239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077476.849244:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.849255:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3182 00000100:00100000:2.0:1769077476.849264:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077476.849269:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.849283:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077476.849326:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913473280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077476.849355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.849358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077476.849360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.849363:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3182 00000100:00080000:0.0:1769077476.849365:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.849369:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077476.849378:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077476.849388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913473408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077476.849396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.849399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077476.849400:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.849408:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3182 00000100:00100000:2.0:1769077476.849411:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913473280:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077476.849412:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.849420:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077476.849436:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077476.849446:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913473536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077476.849453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.849455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077476.849456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.849459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3182 00000100:00080000:0.0:1769077476.849462:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.849465:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077476.849468:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077476.849475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913473664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077476.849481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.849483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077476.849484:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.849486:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3182 00000100:00080000:0.0:1769077476.849489:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.849493:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077476.849496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077476.849502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913473792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077476.849514:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077476.849516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.849522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077476.849523:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.849527:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3182 00000100:00080000:0.0:1769077476.849531:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.849536:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077476.849539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077476.849548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913473920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077476.849559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.849561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077476.849562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.849567:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3182 00000100:00100000:2.0:1769077476.849573:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077476/real 1769077476] req@ffff9350103fb480 x1855008913473280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077476.849574:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.849583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077476.849592:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077476.849609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913474048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077476.849619:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913473280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077476.849633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077476.849643:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077476.849649:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913473280:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077476.849675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913473408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077476.849698:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077476.849710:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077476/real 1769077476] req@ffff9350103fad80 x1855008913473408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077476.849723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913473408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077476.849733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077476.849737:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077476.849740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913473408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077476.849747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913473536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077476.849766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077476.849777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077476/real 1769077476] req@ffff9350103f8700 x1855008913473536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077476.849790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913473536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077476.849799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077476.849803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077476.849807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913473536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077476.849813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913473664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077476.849831:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077476.849841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077476/real 1769077476] req@ffff9350103fb100 x1855008913473664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077476.849854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913473664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077476.849873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077476.849877:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077476.849880:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913473664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077476.849887:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473792:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077476.849906:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077476.849915:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077476/real 1769077476] req@ffff9350103faa00 x1855008913473792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077476.849928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913473792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077476.849945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077476.849949:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077476.849953:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077476.849960:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077476.849977:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077476.849987:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077476/real 1769077476] req@ffff9350103f9c00 x1855008913473920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077476.850000:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913473920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077476.850010:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077476.850013:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077476.850017:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913473920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077476.850023:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077476.850039:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077476.850082:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077476/real 1769077476] req@ffff9350103f8380 x1855008913474048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077476.850111:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913474048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077476.850120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077476.850124:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077476.850127:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077477.041216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077477.553207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077477.553207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077477.553221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077477.553249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077477.873222:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077478.065118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077478.577158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077478.577159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077478.577268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077478.577282:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077478.897187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077479.089175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077479.601168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077479.601168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077479.601180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077479.601201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077479.921202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077480.113123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077480.625152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077480.625152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077480.625154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077480.625185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077480.945206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077481.137145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077481.649154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077481.649166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077481.649244:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077481.650077:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077481.969177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.969190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077481.969196:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077481.969198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077481.969207:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3187 00000100:00080000:3.0:1769077481.969219:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.969227:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077481.969265:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913474176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.969293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.969297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.969300:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.969305:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3187 00000100:00080000:3.0:1769077481.969310:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.969316:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077481.969327:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913474176:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077481.969328:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077481.969346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913474304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.969357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.969361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.969362:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.969367:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3187 00000100:00080000:3.0:1769077481.969371:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.969376:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077481.969381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077481.969391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913474432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077481.969400:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077481.969407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.969411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077481.969414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.969419:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3187 00000100:00080000:3.0:1769077481.969424:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.969428:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077481.969434:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077481.969438:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff935002a7d180 x1855008913474176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077481.969450:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913474560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077481.969458:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913474176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077481.969466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.969475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.969479:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.969488:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3187 00000100:00080000:2.0:1769077481.969493:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077481.969494:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.969504:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077481.969505:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769077481.969512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077481.969514:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913474176:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077481.969530:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913474688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077481.969541:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913474304:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077481.969547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.969552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.969554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.969560:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3187 00000100:00080000:3.0:1769077481.969565:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077481.969566:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077481.969574:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077481.969581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077481.969582:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff935002a7dc00 x1855008913474304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077481.969597:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913474816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077481.969601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913474304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077481.969624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077481.969627:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077481.969630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077481.969634:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077481.969635:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077481.969640:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3187 00000100:00100000:2.0:1769077481.969645:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913474304:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077481.969646:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077481.969654:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913474432:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077481.969663:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077481.969670:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077481.969684:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077481.969686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913474944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077481.969701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff935002a7f800 x1855008913474432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077481.969718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913474432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077481.969732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077481.969736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077481.969738:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913474432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077481.969743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913474560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077481.969759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077481.969767:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff935002a7c000 x1855008913474560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077481.969776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913474560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077481.969782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077481.969784:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077481.969786:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913474560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077481.969799:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474688:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077481.969810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077481.969815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff935002a7df80 x1855008913474688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077481.969823:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913474688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077481.969829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077481.969830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077481.969832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077481.969836:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077481.969846:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077481.969851:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff935002a7fb80 x1855008913474816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077481.969859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913474816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077481.969864:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077481.969866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077481.969877:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077481.969881:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077481.969892:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077481.969899:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff935002a7f100 x1855008913474944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077481.969906:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913474944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077481.969912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077481.969914:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077481.969916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913474944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077482.161125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077482.673141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077482.673141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077482.673142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077482.673180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077482.993175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077483.185108:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077483.697148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077483.697148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077483.697148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077483.697180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077484.017118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077484.209120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077484.721118:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077484.721118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077484.721175:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077484.721188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077485.041156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077485.233117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077485.745159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077485.745159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077485.745159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077485.745194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077486.065131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077486.257111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077486.769127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077486.769148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077486.769149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077486.769181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077487.089148:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077487.089161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077487.089165:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077487.089173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077487.089175:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3192 00000100:00080000:3.0:1769077487.089187:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077487.089195:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077487.089241:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913475072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077487.089275:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077487.089280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077487.089282:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077487.089287:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3192 00000100:00080000:3.0:1769077487.089292:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077487.089298:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077487.089310:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077487.089321:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913475072:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077487.089328:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913475200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077487.089338:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077487.089340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077487.089341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077487.089345:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3192 00000100:00080000:3.0:1769077487.089348:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077487.089352:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077487.089356:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077487.089367:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913475328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077487.089373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077487.089376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077487.089377:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077487.089380:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3192 00000100:00080000:3.0:1769077487.089389:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077487.089392:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077487.089395:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077487.089399:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077487.089407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913475456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077487.089416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077487.089420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077487.089422:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077487.089427:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3192 00000100:00080000:3.0:1769077487.089432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077487.089438:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077487/real 1769077487] req@ffff935002a7c700 x1855008913475072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077487.089439:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077487.089450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077487.089460:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913475072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077487.089470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913475584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077487.089493:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077487.089493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077487.089502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077487.089504:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077487.089507:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077487.089511:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3192 00000100:00080000:3.0:1769077487.089516:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077487.089517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913475072:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077487.089522:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077487.089529:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077487.089539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913475712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077487.089544:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913475200:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077487.089553:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077487.089557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077487.089559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077487.089562:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3192 00000100:00080000:3.0:1769077487.089566:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077487.089568:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077487.089573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077487.089577:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077487.089591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913475840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077487.089593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077487/real 1769077487] req@ffff935002a7e300 x1855008913475200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077487.089627:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913475200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077487.089635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077487.089638:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077487.089642:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913475200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077487.089648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913475328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077487.089661:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077487.089667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077487/real 1769077487] req@ffff935002a7ca80 x1855008913475328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077487.089675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913475328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077487.089681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077487.089684:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077487.089686:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913475328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077487.089690:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913475456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077487.089701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077487.089706:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077487/real 1769077487] req@ffff935002a7c380 x1855008913475456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077487.089714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913475456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077487.089720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077487.089721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077487.089723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913475456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077487.089726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913475584:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077487.089736:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077487.089741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077487/real 1769077487] req@ffff935002a7f100 x1855008913475584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077487.089758:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913475584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077487.089765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077487.089768:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077487.089770:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913475584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077487.089784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913475712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077487.089796:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077487.089803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077487/real 1769077487] req@ffff935002a7fb80 x1855008913475712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077487.089811:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913475712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077487.089817:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077487.089819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077487.089821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913475712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077487.089824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913475840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077487.089832:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077487.089838:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077487/real 1769077487] req@ffff935002a7df80 x1855008913475840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077487.089845:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913475840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077487.089850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077487.089852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077487.089854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913475840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077487.281161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077487.793158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077487.793158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077487.793158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077487.793190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077488.113187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077488.305158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077488.817126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077488.817152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077488.817157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077488.817180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077489.137146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077489.329141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077489.841153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077489.841153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077489.841153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077489.841191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077490.161122:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077490.353148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077490.865122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077490.865132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077490.865148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077490.865304:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077491.185142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077491.377182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077491.889154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077491.889154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077491.889159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077491.889396:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077492.209128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077492.209131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077492.209141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077492.209145:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077492.209156:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3197 00000100:00080000:3.0:1769077492.209168:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077492.209174:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077492.209207:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913475968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077492.209231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077492.209234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077492.209235:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077492.209239:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3197 00000100:00080000:3.0:1769077492.209243:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077492.209246:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077492.209254:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077492.209259:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913475968:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077492.209265:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913476096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077492.209274:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077492.209277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077492.209278:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077492.209281:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3197 00000100:00080000:3.0:1769077492.209284:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077492.209286:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077492.209289:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077492.209298:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913476224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077492.209304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077492.209307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077492.209308:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077492.209311:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3197 00000100:00080000:3.0:1769077492.209314:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077492.209316:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077492.209319:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077492.209323:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077492.209328:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913476352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077492.209340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077492.209343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077492.209344:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077492.209347:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3197 00000100:00080000:3.0:1769077492.209349:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077492.209352:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077492.209356:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077492.209360:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077492/real 1769077492] req@ffff935002a7c000 x1855008913475968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077492.209368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913476480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077492.209377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077492.209382:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913475968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077492.209382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077492.209386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077492.209394:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3197 00000100:00080000:3.0:1769077492.209398:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077492.209403:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077492.209404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769077492.209407:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077492.209410:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077492.209415:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913475968:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077492.209419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913476608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077492.209426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077492.209429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077492.209430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077492.209432:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3197 00000100:00080000:3.0:1769077492.209436:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077492.209438:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077492.209443:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476096:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077492.209444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077492.209453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913476736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077492.209488:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077492.209505:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077492/real 1769077492] req@ffff935002a7f800 x1855008913476096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077492.209517:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913476096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077492.209523:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077492.209526:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077492.209528:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077492.209533:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077492.209549:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077492.209560:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077492/real 1769077492] req@ffff935002a7dc00 x1855008913476224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077492.209580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913476224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077492.209589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077492.209593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077492.209597:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077492.209603:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077492.209622:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077492.209633:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077492/real 1769077492] req@ffff935002a7d180 x1855008913476352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077492.209646:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913476352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077492.209655:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077492.209659:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077492.209662:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077492.209669:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913476480:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077492.209690:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077492.209701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077492/real 1769077492] req@ffff935002a7df80 x1855008913476480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077492.209714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913476480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077492.209729:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077492.209733:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077492.209736:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913476480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077492.209741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913476608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077492.209757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077492.209765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077492/real 1769077492] req@ffff935002a7fb80 x1855008913476608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077492.209773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913476608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077492.209779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077492.209781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077492.209783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913476608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077492.209798:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913476736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077492.209808:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077492.209813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077492/real 1769077492] req@ffff935002a7f100 x1855008913476736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077492.209830:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913476736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077492.209849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077492.209853:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077492.209856:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913476736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077492.401162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077492.913134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077492.913146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077492.913261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077492.913269:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077493.233180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077493.425152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077493.937154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077493.937155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077493.937169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077493.937202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077494.257156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077494.449147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077494.961126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077494.961126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077494.961126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077494.961158:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077495.281124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077495.473135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077495.985215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077495.985215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077495.985215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077495.985290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077496.305199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077496.497169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077497.009196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077497.009197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077497.009345:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077497.009358:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077497.329173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077497.329192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077497.329199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077497.329204:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077497.329212:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3203 00000100:00080000:2.0:1769077497.329224:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077497.329232:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077497.329282:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913476864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077497.329310:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077497.329313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077497.329314:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077497.329318:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3203 00000100:00080000:2.0:1769077497.329321:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077497.329326:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077497.329337:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077497.329346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913476864:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077497.329351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913476992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077497.329360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077497.329363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077497.329364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077497.329366:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3203 00000100:00080000:2.0:1769077497.329369:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077497.329372:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077497.329378:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077497.329390:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913477120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077497.329400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077497.329404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077497.329407:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077497.329411:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3203 00000100:00080000:2.0:1769077497.329416:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077497.329421:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077497.329426:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077497.329429:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077497.329443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913477248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077497.329455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077497.329459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077497.329461:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077497.329465:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3203 00000100:00080000:2.0:1769077497.329471:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077497.329477:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077497.329484:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077497/real 1769077497] req@ffff935002a62680 x1855008913476864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077497.329484:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077497.329506:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913477376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077497.329511:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913476864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077497.329519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077497.329527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077497.329530:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077497.329533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077497.329534:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3203 00000100:00080000:2.0:1769077497.329540:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077497.329545:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077497.329546:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077497.329554:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077497.329555:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913476864:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077497.329566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913477504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077497.329574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077497.329576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077497.329578:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077497.329580:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3203 00000100:00080000:2.0:1769077497.329581:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077497.329584:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077497.329588:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077497.329590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077497.329597:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913477632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077497.329620:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077497.329635:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077497/real 1769077497] req@ffff935002a61f80 x1855008913476992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077497.329647:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913476992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077497.329657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077497.329660:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077497.329664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913476992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077497.329670:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913477120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077497.329689:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077497.329701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077497/real 1769077497] req@ffff935002a61c00 x1855008913477120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077497.329713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913477120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077497.329722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077497.329726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077497.329729:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913477120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077497.329735:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913477248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077497.329766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077497.329778:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077497/real 1769077497] req@ffff935002a63100 x1855008913477248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077497.329790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913477248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077497.329800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077497.329803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077497.329806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913477248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077497.329812:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913477376:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077497.329829:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077497.329838:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077497/real 1769077497] req@ffff935002a60a80 x1855008913477376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077497.329850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913477376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077497.329858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077497.329862:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077497.329865:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913477376:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077497.329871:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913477504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077497.329892:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077497.329902:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077497/real 1769077497] req@ffff935002a61180 x1855008913477504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077497.329921:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913477504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077497.329931:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077497.329934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077497.329938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913477504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077497.329957:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913477632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077497.329980:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077497.329990:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077497/real 1769077497] req@ffff935002a61880 x1855008913477632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077497.330003:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913477632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077497.330012:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077497.330015:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077497.330018:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913477632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077497.521134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077498.033173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077498.033190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077498.033209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077498.033235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077498.353181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077498.545190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077499.057179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077499.057180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077499.057180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077499.057217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077499.377145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077499.569160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077500.081186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077500.081187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077500.081198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077500.081296:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077500.401201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077500.593178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077501.105186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077501.105187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077501.105192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077501.105268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077501.425149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077501.617164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077502.129158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077502.129183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077502.129184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077502.129218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077502.449186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077502.449188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077502.449201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077502.449207:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077502.449219:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3208 00000100:00080000:2.0:1769077502.449234:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077502.449244:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077502.449292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913477760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077502.449327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077502.449331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077502.449334:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077502.449339:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3208 00000100:00080000:2.0:1769077502.449343:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077502.449349:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077502.449362:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913477760:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769077502.449362:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077502.449381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913477888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077502.449392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077502.449396:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077502.449398:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077502.449403:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3208 00000100:00080000:2.0:1769077502.449407:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077502.449412:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077502.449418:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077502.449429:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913478016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077502.449440:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077502.449443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077502.449449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077502.449451:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077502.449457:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3208 00000100:00080000:2.0:1769077502.449465:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077502.449470:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077502.449476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077502.449486:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077502/real 1769077502] req@ffff935002a62300 x1855008913477760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077502.449494:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913478144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077502.449508:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077502.449513:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913477760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077502.449514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077502.449519:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077502.449529:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3208 00000100:00080000:0.0:1769077502.449535:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077502.449536:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077502.449545:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077502.449551:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913477760:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077502.449558:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077502.449565:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077502.449580:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913477888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077502.449584:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913478272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077502.449597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077502.449602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077502.449605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077502.449609:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3208 00000800:00000100:0.0:1769077502.449614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077502.449616:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077502.449624:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077502.449630:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077502.449636:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077502/real 1769077502] req@ffff935002a60000 x1855008913477888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077502.449646:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913478400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077502.449657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913477888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077502.449688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077502.449694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077502.449702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913477888:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077502.449703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077502.449711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077502.449713:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478016:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769077502.449715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077502.449723:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3208 00000100:00080000:2.0:1769077502.449729:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077502.449734:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077502.449740:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077502.449742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077502.449759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913478528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077502.449762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077502/real 1769077502] req@ffff935002a62a00 x1855008913478016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077502.449799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913478016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077502.449806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077502.449809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077502.449811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077502.449816:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077502.449833:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077502.449841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077502/real 1769077502] req@ffff935002a61500 x1855008913478144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077502.449848:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913478144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077502.449854:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077502.449856:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077502.449859:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077502.449871:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913478272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077502.449881:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077502.449888:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077502/real 1769077502] req@ffff935002a62300 x1855008913478272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077502.449895:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913478272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077502.449906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077502.449908:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077502.449911:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913478272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077502.449914:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913478400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077502.449923:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077502.449928:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077502/real 1769077502] req@ffff935002a61880 x1855008913478400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077502.449935:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913478400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077502.449941:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077502.449943:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077502.449945:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913478400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077502.449948:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913478528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077502.449957:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077502.449963:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077502/real 1769077502] req@ffff935002a61180 x1855008913478528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077502.449970:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913478528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077502.449975:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077502.449977:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077502.449980:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913478528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077502.641133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077503.153138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077503.153146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077503.153168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077503.153182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077503.473168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077503.665117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077504.177127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077504.177127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077504.177138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077504.177251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077504.497164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077504.689135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077505.201168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077505.201168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077505.201168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077505.201240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077505.521164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077505.713152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077506.225117:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077506.225142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077506.225238:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077506.225268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077506.545241:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077506.737133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077507.249177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077507.249177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077507.249232:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077507.249240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077507.569170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077507.569181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077507.569194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077507.569200:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077507.569212:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3213 00000100:00080000:0.0:1769077507.569223:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077507.569229:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077507.569269:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913478656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077507.569300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077507.569303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077507.569305:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077507.569312:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3213 00000100:00080000:0.0:1769077507.569318:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077507.569324:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077507.569336:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077507.569338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913478656:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077507.569360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913478784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077507.569377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077507.569381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077507.569384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077507.569390:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3213 00000100:00080000:0.0:1769077507.569397:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077507.569402:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077507.569409:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077507.569427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913478912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077507.569433:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077507.569447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077507.569450:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077507.569452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077507.569458:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3213 00000100:00080000:0.0:1769077507.569464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077507.569468:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077507.569473:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077507.569486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913479040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077507.569493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077507.569496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077507.569502:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077507.569502:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077507/real 1769077507] req@ffff9350103fbb80 x1855008913478656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077507.569516:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3213 00000100:00080000:0.0:1769077507.569524:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077507.569531:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913478656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077507.569533:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077507.569551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077507.569552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077507.569559:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077507.569563:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913478656:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077507.569574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913479168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077507.569596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077507.569599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077507.569600:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077507.569604:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3213 00000100:00080000:0.0:1769077507.569607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077507.569609:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478784:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077507.569617:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077507.569626:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077507.569641:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913479296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077507.569651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077507.569660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077507.569665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077507.569668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077507.569672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077507/real 1769077507] req@ffff9350103fb800 x1855008913478784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077507.569678:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3213 00000100:00080000:0.0:1769077507.569691:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077507.569700:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077507.569703:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913478784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077507.569707:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077507.569725:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077507.569728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913479424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077507.569734:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077507.569749:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077507.569775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077507.569795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077507.569807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077507/real 1769077507] req@ffff9350103f9f80 x1855008913478912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077507.569815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913478912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077507.569820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077507.569822:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077507.569824:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913478912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077507.569828:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077507.569838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077507.569844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077507/real 1769077507] req@ffff9350103f9180 x1855008913479040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077507.569854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913479040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077507.569862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077507.569866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077507.569868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077507.569875:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913479168:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077507.569897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077507.569910:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077507/real 1769077507] req@ffff9350103f9500 x1855008913479168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077507.569923:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913479168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077507.569932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077507.569934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077507.569937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913479168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077507.569954:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913479296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077507.569970:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077507.569977:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077507/real 1769077507] req@ffff9350103fbb80 x1855008913479296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077507.569985:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913479296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077507.569990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077507.569992:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077507.569995:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913479296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077507.569999:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913479424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077507.570009:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077507.570014:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077507/real 1769077507] req@ffff9350103f8380 x1855008913479424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077507.570021:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913479424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077507.570026:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077507.570028:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077507.570029:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913479424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077507.761112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077508.273120:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077508.273141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077508.273143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077508.273223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077508.593242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077508.785192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077509.297159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077509.297159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077509.297167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077509.297221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077509.617150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077509.809220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077510.321197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077510.321206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077510.321210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077510.321283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077510.641173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077510.833134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077511.345130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077511.345131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077511.345136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077511.345253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077511.665121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077511.857177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077512.369187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077512.369188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077512.369208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077512.369224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077512.689145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077512.689158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077512.689163:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077512.689174:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3218 00000100:00100000:2.0:1769077512.689174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077512.689190:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077512.689198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077512.689235:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913479552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077512.689259:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077512.689261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077512.689263:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077512.689266:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3218 00000100:00080000:0.0:1769077512.689268:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077512.689271:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077512.689278:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077512.689289:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913479680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077512.689299:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913479552:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077512.689302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077512.689308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077512.689310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077512.689314:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3218 00000100:00080000:0.0:1769077512.689316:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077512.689318:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077512.689322:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077512.689330:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913479808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077512.689337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077512.689340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077512.689341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077512.689343:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3218 00000100:00080000:0.0:1769077512.689345:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077512.689347:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077512.689350:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077512.689358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913479936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077512.689365:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077512.689371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077512.689374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077512.689376:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077512.689380:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3218 00000100:00080000:0.0:1769077512.689384:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077512.689387:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077512.689390:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077512.689400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913480064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077512.689406:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077512/real 1769077512] req@ffff9350103f9c00 x1855008913479552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077512.689409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077512.689417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077512.689420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077512.689427:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3218 00000100:00100000:2.0:1769077512.689436:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913479552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077512.689438:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077512.689449:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077512.689450:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769077512.689455:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077512.689458:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077512.689463:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913479552:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077512.689468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913480192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077512.689475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077512.689478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077512.689479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077512.689481:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3218 00000100:00080000:0.0:1769077512.689484:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077512.689488:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479680:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077512.689489:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077512.689512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077512.689524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913480320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077512.689538:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077512.689551:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077512/real 1769077512] req@ffff9350103faa00 x1855008913479680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077512.689562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913479680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077512.689569:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077512.689572:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077512.689574:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077512.689579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077512.689591:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077512.689597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077512/real 1769077512] req@ffff9350103fb100 x1855008913479808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077512.689605:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913479808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077512.689611:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077512.689613:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077512.689615:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077512.689618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077512.689628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077512.689634:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077512/real 1769077512] req@ffff9350103f8700 x1855008913479936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077512.689647:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913479936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077512.689652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077512.689654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077512.689656:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913479936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077512.689660:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480064:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077512.689670:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077512.689676:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077512/real 1769077512] req@ffff9350103fad80 x1855008913480064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077512.689683:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913480064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077512.689688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077512.689690:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077512.689692:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077512.689695:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077512.689704:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077512.689709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077512/real 1769077512] req@ffff9350103fb480 x1855008913480192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077512.689716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913480192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077512.689725:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077512.689728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077512.689730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077512.689743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077512.689752:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077512.689758:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077512/real 1769077512] req@ffff9350103f9c00 x1855008913480320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077512.689765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913480320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077512.689771:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077512.689773:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077512.689775:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077512.881139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077513.393157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077513.393157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077513.393215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077513.393259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077513.713184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077513.905169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077514.417135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077514.417160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077514.417179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077514.417181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077514.737147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077514.929122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077515.441165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077515.441165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077515.441165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077515.441202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077515.761117:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077515.953143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077516.465224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077516.465224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077516.465275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077516.465379:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077516.785131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077516.977130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077517.489167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077517.489167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077517.489173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077517.489214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077517.809179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077517.809188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077517.809193:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077517.809194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077517.809204:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3223 00000100:00080000:0.0:1769077517.809215:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077517.809221:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077517.809255:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913480448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077517.809280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077517.809283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077517.809284:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077517.809288:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3223 00000100:00080000:0.0:1769077517.809290:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077517.809292:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077517.809301:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077517.809311:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913480576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077517.809320:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913480448:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077517.809322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077517.809329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077517.809330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077517.809334:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3223 00000100:00080000:0.0:1769077517.809337:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077517.809339:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077517.809343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077517.809353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913480704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077517.809360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077517.809363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077517.809364:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077517.809367:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3223 00000100:00080000:0.0:1769077517.809369:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077517.809371:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077517.809375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077517.809383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913480832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077517.809389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077517.809392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077517.809393:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077517.809396:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3223 00000100:00080000:0.0:1769077517.809399:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077517.809403:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077517.809423:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077517.809435:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913480960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077517.809442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077517.809445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077517.809447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077517.809450:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3223 00000100:00100000:2.0:1769077517.809454:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913480576:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077517.809455:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077517.809462:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077517.809467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077517.809478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913481088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077517.809485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077517.809489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077517.809490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077517.809493:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3223 00000100:00080000:0.0:1769077517.809496:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077517.809500:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913480704:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077517.809501:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077517.809508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077517.809519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913481216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077517.809538:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913480832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077517.809558:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077517.809576:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077517.809588:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077518.001161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077518.513164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077518.513166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077518.513166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077518.513212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077518.833210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077519.025115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077519.537164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077519.537164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077519.537177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077519.537202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077519.857159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077520.049155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077520.561164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077520.561164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077520.561179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077520.561232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077520.881155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077520.881326:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077520.881386:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077520.881405:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077520.881456:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077520.881466:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077520.881475:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077520.881483:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077520.881493:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077520.881503:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077520.881597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077517/real 1769077520] req@ffff9350103fb800 x1855008913481088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077520.881630:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913481088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077520.881645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077520.881652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077520.881658:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077520.881667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077517/real 1769077520] req@ffff9350103f9c00 x1855008913481216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077520.881680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913481216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077520.881688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077520.881692:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077520.881696:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077520.881702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077517/real 1769077520] req@ffff9350103fbb80 x1855008913480576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077520.881722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913480576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077520.881731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077520.881734:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077520.881737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913480576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077520.881745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077517/real 1769077520] req@ffff9350103f9500 x1855008913480704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077520.881757:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913480704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077520.881764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077520.881769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077520.881772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913480704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077520.881779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077517/real 1769077520] req@ffff9350103f9180 x1855008913480832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077520.881792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913480832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077520.881800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077520.881804:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077520.881807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913480832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077520.881814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077517/real 1769077520] req@ffff9350103f9f80 x1855008913480960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077520.881825:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913480960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077520.881834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077520.881837:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077520.881841:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913480960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077520.881847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077517/real 1769077520] req@ffff9350103f8380 x1855008913480448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077520.881861:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913480448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077520.881868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077520.881871:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077520.881874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913480448:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077521.073137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077521.585149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077521.585150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077521.585153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077521.585196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077521.905161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077522.097157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077522.609163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077522.609163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077522.609163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077522.609210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077522.929139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077522.929159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.929168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077522.929173:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.929183:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3228 00000100:00080000:1.0:1769077522.929195:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.929203:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077522.929240:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913481344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.929269:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.929273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077522.929275:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.929280:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3228 00000100:00080000:1.0:1769077522.929285:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.929290:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077522.929299:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077522.929315:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913481472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.929343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077522.929347:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913481344:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077522.929348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077522.929352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.929362:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3228 00000100:00080000:1.0:1769077522.929367:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.929371:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077522.929378:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077522.929393:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913481600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.929404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.929408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077522.929410:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.929414:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3228 00000100:00080000:1.0:1769077522.929419:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.929424:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077522.929429:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077522.929441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913481728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.929452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.929458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077522.929460:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.929466:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3228 00000800:00000100:2.0:1769077522.929469:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077522.929472:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.929478:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077522.929485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077522.929497:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913481856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.929508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.929513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077522.929515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.929520:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3228 00000100:00080000:1.0:1769077522.929524:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.929528:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077522.929534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077522.929545:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9350100c9f80 x1855008913481344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077522.929550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913481984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.929569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.929572:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077522.929575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.929581:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3228 00000100:00080000:1.0:1769077522.929587:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.929592:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077522.929598:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077522.929600:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913481344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077522.929613:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913482112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077522.929674:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077522.929693:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077522.929698:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913481344:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077522.929750:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913481472:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077522.929786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077522.929802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9350100c8000 x1855008913481472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077522.929818:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913481472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077522.929828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077522.929831:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077522.929835:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913481472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077522.929844:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913481600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077522.929868:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077522.929904:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9350100c8a80 x1855008913481600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077522.929921:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913481600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077522.929929:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077522.929932:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077522.929935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913481600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077522.929940:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913481728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077522.929966:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077522.929979:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9350100c9180 x1855008913481728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077522.929992:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913481728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077522.930002:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077522.930006:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077522.930011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913481728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077522.930018:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481856:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077522.930039:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077522.930091:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9350100cb100 x1855008913481856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077522.930104:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913481856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077522.930113:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077522.930117:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077522.930120:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077522.930126:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077522.930147:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077522.930159:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9350100c9c00 x1855008913481984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077522.930178:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913481984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077522.930186:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077522.930190:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077522.930193:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913481984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077522.930200:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913482112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077522.930218:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077522.930230:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9350100cad80 x1855008913482112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077522.930241:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913482112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077522.930249:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077522.930253:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077522.930257:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913482112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077523.121154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077523.633124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077523.633125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077523.633131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077523.633179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077523.953193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077524.145162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077524.657174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077524.657176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077524.657188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077524.657277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077524.977144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077525.169131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077525.681136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077525.681136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077525.681143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077525.681169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077526.001176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077526.193130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077526.705177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077526.705177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077526.705209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077526.705351:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077527.025107:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077527.217158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077527.729137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077527.729137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077527.729160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077527.729173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077528.049207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077528.049218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077528.049248:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077528.049255:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077528.049266:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3233 00000100:00080000:3.0:1769077528.049280:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077528.049287:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077528.049332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913482240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077528.049366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077528.049371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077528.049373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077528.049379:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3233 00000100:00080000:3.0:1769077528.049383:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077528.049389:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077528.049402:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077528.049418:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913482240:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077528.049423:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913482368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077528.049442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077528.049446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077528.049448:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077528.049455:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3233 00000100:00080000:3.0:1769077528.049459:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077528.049463:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077528.049470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077528.049481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913482496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077528.049492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077528.049496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077528.049499:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077528.049505:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077528.049511:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3233 00000100:00080000:3.0:1769077528.049519:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077528.049524:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077528.049532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077528.049548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913482624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077528.049563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077528.049565:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077528/real 1769077528] req@ffff935002a7c380 x1855008913482240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077528.049574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077528.049583:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077528.049603:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3233 00000100:00080000:3.0:1769077528.049611:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077528.049615:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913482240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077528.049618:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077528.049633:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077528.049635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077528.049643:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077528.049647:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913482240:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077528.049659:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913482752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077528.049671:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913482368:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077528.049673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077528.049682:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077528.049685:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077528.049692:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3233 00000100:00080000:3.0:1769077528.049698:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077528.049699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077528.049707:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077528.049718:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077528.049722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077528/real 1769077528] req@ffff935002a7ca80 x1855008913482368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077528.049742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913482368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077528.049745:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913482880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077528.049778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077528.049788:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077528.049789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077528.049801:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077528.049805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913482368:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077528.049806:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077528.049822:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3233 00000100:00100000:2.0:1769077528.049828:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913482496:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077528.049830:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077528.049838:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077528.049844:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077528.049850:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077528.049862:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077528/real 1769077528] req@ffff935002a7e300 x1855008913482496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077528.049869:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913483008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077528.049887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913482496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077528.049903:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077528.049906:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077528.049908:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913482496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077528.049915:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913482624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077528.049929:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077528.049935:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077528/real 1769077528] req@ffff935002a7c700 x1855008913482624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077528.049942:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913482624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077528.049948:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077528.049950:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077528.049952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913482624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077528.049964:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913482752:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077528.049975:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077528.049981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077528/real 1769077528] req@ffff935002a7f100 x1855008913482752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077528.049988:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913482752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077528.049994:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077528.049996:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077528.049997:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913482752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077528.050002:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913482880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077528.050011:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077528.050016:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077528/real 1769077528] req@ffff935002a7fb80 x1855008913482880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077528.050029:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913482880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077528.050037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077528.050041:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077528.050044:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913482880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077528.050083:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077528.050104:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077528.050121:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077528/real 1769077528] req@ffff935002a7df80 x1855008913483008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077528.050130:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913483008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077528.050136:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077528.050138:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077528.050140:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077528.241149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077528.753241:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077528.753241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077528.753294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077528.753308:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077529.073187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077529.265185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077529.777142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077529.777145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077529.777146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077529.777272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077530.097234:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077530.289155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077530.801146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077530.801146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077530.801179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077530.801230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077531.121287:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077531.313211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077531.825124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077531.825149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077531.825201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077531.825251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077532.145220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077532.337128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077532.849170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077532.849170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077532.849183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077532.849211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077533.169156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077533.169197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077533.169211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077533.169223:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077533.169234:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3238 00000100:00080000:3.0:1769077533.169246:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077533.169254:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077533.169293:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913483136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077533.169323:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077533.169327:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077533.169329:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077533.169333:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3238 00000100:00080000:3.0:1769077533.169337:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077533.169341:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077533.169351:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077533.169365:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913483264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077533.169377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077533.169381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077533.169385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077533.169388:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913483136:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077533.169395:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3238 00000100:00080000:3.0:1769077533.169405:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077533.169410:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077533.169417:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077533.169432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913483392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077533.169450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077533.169454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077533.169456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077533.169461:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3238 00000100:00080000:3.0:1769077533.169478:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077533.169484:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077533.169490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077533.169492:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077533.169512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913483520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077533.169525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077533.169529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077533.169531:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077533.169536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3238 00000100:00080000:3.0:1769077533.169541:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077533.169548:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077533.169554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077533.169559:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077533/real 1769077533] req@ffff935002a7d180 x1855008913483136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077533.169574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913483648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077533.169587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077533.169588:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913483136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077533.169598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077533.169607:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077533.169619:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3238 00000100:00080000:2.0:1769077533.169625:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077533.169626:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077533.169638:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077533.169639:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077533.169652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913483136:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077533.169653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077533.169678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913483776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077533.169690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077533.169693:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077533.169698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077533.169699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913483264:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077533.169713:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3238 00000100:00080000:3.0:1769077533.169720:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077533.169726:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077533.169742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077533.169743:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077533.169762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077533/real 1769077533] req@ffff935002a7dc00 x1855008913483264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077533.169768:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913483904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077533.169799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913483264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077533.169810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077533.169816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077533.169820:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913483264:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077533.169828:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913483392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077533.169850:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077533.169862:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077533/real 1769077533] req@ffff935002a7f800 x1855008913483392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077533.169873:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913483392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077533.169881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077533.169885:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077533.169888:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913483392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077533.169894:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913483520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077533.169913:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077533.169923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077533/real 1769077533] req@ffff935002a7c000 x1855008913483520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077533.169934:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913483520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077533.169942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077533.169945:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077533.169948:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913483520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077533.169954:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483648:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077533.169974:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077533.169984:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077533/real 1769077533] req@ffff935002a7df80 x1855008913483648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077533.169996:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913483648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077533.170004:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077533.170007:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077533.170010:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077533.170017:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077533.170036:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077533.170047:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077533/real 1769077533] req@ffff935002a7fb80 x1855008913483776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077533.170096:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913483776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077533.170105:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077533.170108:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077533.170111:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077533.170136:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077533.170175:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077533.170188:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077533/real 1769077533] req@ffff935002a7f100 x1855008913483904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077533.170200:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913483904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077533.170208:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077533.170211:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077533.170215:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913483904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077533.361170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077533.873132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077533.873190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077533.873216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077533.873238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077534.193144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077534.385124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077534.897229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077534.897236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077534.897251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077534.897276:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077535.217193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077535.409148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077535.921215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077535.921216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077535.921265:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077535.921294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077536.241177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077536.433179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077536.945179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077536.945179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077536.945230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077536.945236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077537.265143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077537.457166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077537.969216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077537.969216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077537.969224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077537.969282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077538.289142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077538.289150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077538.289158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077538.289163:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077538.289174:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3243 00000100:00080000:3.0:1769077538.289186:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077538.289193:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077538.289235:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913484032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077538.289263:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077538.289265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077538.289267:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077538.289270:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3243 00000100:00080000:3.0:1769077538.289273:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077538.289276:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077538.289285:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077538.289299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913484160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077538.289310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077538.289314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077538.289316:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077538.289321:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3243 00000100:00080000:3.0:1769077538.289325:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077538.289330:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077538.289337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077538.289350:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913484032:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077538.289353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913484288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077538.289385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077538.289387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077538.289389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077538.289393:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3243 00000100:00080000:3.0:1769077538.289397:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077538.289401:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077538.289405:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077538.289415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913484416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077538.289422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077538.289425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077538.289427:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077538.289431:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3243 00000100:00080000:3.0:1769077538.289452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077538.289455:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077538.289459:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077538.289467:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077538.289468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913484544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077538.289479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077538.289483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077538.289484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077538.289488:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3243 00000100:00080000:3.0:1769077538.289494:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077538.289500:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077538.289504:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077538.289513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913484672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077538.289520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077538.289523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077538.289525:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077538.289530:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3243 00000100:00080000:3.0:1769077538.289534:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077538.289539:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077538/real 1769077538] req@ffff935002a7c700 x1855008913484032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077538.289541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077538.289550:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077538.289564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913484800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077538.289634:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913484032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077538.289653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077538.289659:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077538.289664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913484032:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077538.289673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913484160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077538.289698:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077538.289711:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077538/real 1769077538] req@ffff935002a7e300 x1855008913484160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077538.289724:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913484160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077538.289733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077538.289736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077538.289740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913484160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077538.289772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913484288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077538.289791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077538.289802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077538/real 1769077538] req@ffff935002a7ca80 x1855008913484288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077538.289814:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913484288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077538.289829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077538.289833:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077538.289837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913484288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077538.289843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913484416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077538.289858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077538.289868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077538/real 1769077538] req@ffff935002a7c380 x1855008913484416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077538.289879:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913484416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077538.289888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077538.289900:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077538.289904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913484416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077538.289910:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913484544:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077538.289928:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077538.289938:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077538/real 1769077538] req@ffff935002a7f100 x1855008913484544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077538.289950:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913484544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077538.289958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077538.289961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077538.289965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913484544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077538.289972:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913484672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077538.289987:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077538.289996:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077538/real 1769077538] req@ffff935002a7fb80 x1855008913484672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077538.290008:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913484672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077538.290016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077538.290019:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077538.290023:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913484672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077538.290029:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913484800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077538.290043:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077538.290108:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077538/real 1769077538] req@ffff935002a7df80 x1855008913484800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077538.290121:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913484800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077538.290130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077538.290133:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077538.290136:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913484800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077538.481161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077538.993224:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077538.993225:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077538.993269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077538.993275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077539.313212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077539.505152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077540.017158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077540.017158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077540.017158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077540.017287:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077540.337123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077540.529160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077541.041139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077541.041139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077541.041171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077541.041237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077541.361181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077541.553162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077542.065157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077542.065177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077542.065190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077542.065256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077542.385180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077542.577150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077543.089125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077543.089138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077543.089174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077543.089276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077543.409165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077543.409165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077543.409174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077543.409179:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077543.409187:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3249 00000100:00080000:1.0:1769077543.409197:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077543.409204:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077543.409239:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913484928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077543.409264:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077543.409266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077543.409268:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077543.409271:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3249 00000100:00080000:1.0:1769077543.409273:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077543.409275:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077543.409282:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077543.409291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913485056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077543.409301:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077543.409305:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077543.409307:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077543.409312:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3249 00000100:00080000:1.0:1769077543.409316:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077543.409322:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077543.409324:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913484928:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077543.409328:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077543.409344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913485184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077543.409363:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077543.409367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077543.409370:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077543.409375:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3249 00000100:00080000:1.0:1769077543.409379:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077543.409382:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077543.409386:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077543.409396:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913485312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077543.409401:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077543.409409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077543.409413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077543.409415:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077543.409419:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3249 00000100:00080000:1.0:1769077543.409423:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077543.409425:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077543.409429:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077543.409437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913485440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077543.409445:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077543/real 1769077543] req@ffff9350100ca680 x1855008913484928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077543.409446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077543.409461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077543.409463:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077543.409465:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913484928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077543.409469:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3249 00000100:00080000:1.0:1769077543.409474:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077543.409480:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077543.409485:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769077543.409487:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077543.409491:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077543.409496:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913484928:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077543.409506:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077543.409507:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913485568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077543.409522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077543.409525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077543.409527:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077543.409528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077543.409532:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3249 00000100:00080000:1.0:1769077543.409537:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077543.409541:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077543.409544:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077543/real 1769077543] req@ffff9350100c8e00 x1855008913485056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077543.409547:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077543.409560:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913485056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077543.409563:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913485696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077543.409576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077543.409587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077543.409592:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077543.409621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077543.409640:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077543.409648:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077543/real 1769077543] req@ffff9350100c9880 x1855008913485184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077543.409671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913485184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077543.409677:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077543.409679:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077543.409682:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077543.409686:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077543.409698:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077543.409705:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077543/real 1769077543] req@ffff9350100c9500 x1855008913485312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077543.409718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913485312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077543.409726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077543.409729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077543.409733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077543.409739:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913485440:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077543.409759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077543.409770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077543/real 1769077543] req@ffff9350100caa00 x1855008913485440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077543.409783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913485440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077543.409794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077543.409797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077543.409799:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913485440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077543.409803:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913485568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077543.409818:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077543.409824:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077543/real 1769077543] req@ffff9350100c8380 x1855008913485568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077543.409832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913485568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077543.409840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077543.409844:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077543.409847:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913485568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077543.409854:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913485696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077543.409882:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077543.409894:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077543/real 1769077543] req@ffff9350100ca300 x1855008913485696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077543.409903:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913485696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077543.409909:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077543.409911:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077543.409914:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913485696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077543.601137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077544.113175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077544.113177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077544.113200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077544.113213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077544.433161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077544.625201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077545.137159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077545.137191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077545.137194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077545.137241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077545.457146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077545.649174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077546.161152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077546.161152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077546.161197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077546.161267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077546.481135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077546.673166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077547.185150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077547.185150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077547.185157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077547.185285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077547.505157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077547.697127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077548.209151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077548.209154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077548.209158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077548.209234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077548.529161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077548.529170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077548.529171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077548.529177:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077548.529186:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3254 00000100:00080000:0.0:1769077548.529196:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077548.529203:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077548.529242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913485824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077548.529263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077548.529266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077548.529267:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077548.529270:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3254 00000100:00080000:0.0:1769077548.529273:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077548.529275:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077548.529283:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077548.529292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913485952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077548.529302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077548.529304:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077548.529306:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077548.529311:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3254 00000100:00080000:0.0:1769077548.529314:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077548.529322:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077548.529322:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913485824:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769077548.529338:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077548.529349:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913486080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077548.529359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077548.529362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077548.529364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077548.529367:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3254 00000100:00080000:0.0:1769077548.529370:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077548.529373:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077548.529376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077548.529385:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913486208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077548.529391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077548.529393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077548.529395:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077548.529399:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3254 00000100:00080000:0.0:1769077548.529402:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077548.529404:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077548.529407:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077548.529415:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077548.529422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913486336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077548.529432:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077548.529436:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077548.529437:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077548.529441:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3254 00000100:00080000:0.0:1769077548.529444:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077548.529447:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077548.529452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077548.529465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913486464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077548.529471:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077548/real 1769077548] req@ffff9350103fb480 x1855008913485824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077548.529488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077548.529492:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077548.529495:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077548.529495:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913485824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077548.529514:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3254 00000100:00080000:0.0:1769077548.529528:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077548.529531:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077548.529536:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077548.529541:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:0.0:1769077548.529543:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077548.529549:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913485824:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077548.529559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913486592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077548.529583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077548.529602:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077548.529611:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077548/real 1769077548] req@ffff9350103fad80 x1855008913485952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077548.529620:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913485952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077548.529627:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077548.529629:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077548.529632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913485952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077548.529636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077548.529648:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077548.529654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077548/real 1769077548] req@ffff9350103f8700 x1855008913486080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077548.529661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913486080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077548.529666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077548.529668:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077548.529671:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077548.529675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077548.529684:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077548.529691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077548/real 1769077548] req@ffff9350103fb100 x1855008913486208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077548.529702:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913486208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077548.529710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077548.529713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077548.529717:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077548.529724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913486336:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077548.529745:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077548.529757:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077548/real 1769077548] req@ffff9350103faa00 x1855008913486336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077548.529767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913486336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077548.529772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077548.529774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077548.529777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913486336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077548.529781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913486464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077548.529795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077548.529802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077548/real 1769077548] req@ffff9350103f8380 x1855008913486464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077548.529809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913486464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077548.529814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077548.529816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077548.529818:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913486464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077548.529822:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913486592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077548.529831:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077548.529837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077548/real 1769077548] req@ffff9350103f9f80 x1855008913486592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077548.529844:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913486592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077548.529849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077548.529851:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077548.529854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913486592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077548.721147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077549.233183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077549.233183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077549.233189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077549.233258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077549.553199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077549.745108:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077550.257158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077550.257158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077550.257158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077550.257182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077550.577160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077550.769172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077551.281147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077551.281147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077551.281156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077551.281199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077551.601130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077551.793148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077552.305221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077552.305223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077552.305224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077552.305258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077552.625252:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077552.817180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077553.329147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077553.329164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077553.329175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077553.329181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077553.649161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077553.649165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077553.649174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077553.649179:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077553.649197:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3259 00000100:00080000:2.0:1769077553.649207:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077553.649214:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077553.649246:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913486720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077553.649269:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077553.649272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077553.649273:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077553.649277:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3259 00000100:00080000:2.0:1769077553.649279:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077553.649281:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077553.649289:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077553.649299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913486848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077553.649307:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077553.649310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077553.649311:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077553.649314:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3259 00000100:00080000:2.0:1769077553.649316:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077553.649318:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077553.649321:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077553.649326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913486720:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077553.649330:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913486976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077553.649342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077553.649344:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077553.649346:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077553.649348:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3259 00000100:00080000:2.0:1769077553.649351:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077553.649353:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077553.649356:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077553.649364:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913487104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077553.649370:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077553.649372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077553.649373:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077553.649376:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3259 00000100:00080000:2.0:1769077553.649380:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077553.649382:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077553.649385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077553.649394:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913487232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077553.649403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077553.649410:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077553.649411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077553.649415:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077553.649421:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3259 00000100:00080000:2.0:1769077553.649427:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077553.649432:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077553.649438:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077553.649454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913487360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077553.649463:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077553/real 1769077553] req@ffff935002a60a80 x1855008913486720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077553.649464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077553.649470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077553.649472:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077553.649479:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3259 00000100:00080000:2.0:1769077553.649485:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077553.649489:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077553.649494:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077553.649500:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913486720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077553.649505:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913487488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077553.649528:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077553.649539:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077553.649545:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913486720:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077553.649554:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486848:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077553.649581:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077553.649613:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077553/real 1769077553] req@ffff935002a63100 x1855008913486848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077553.649632:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913486848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077553.649644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077553.649649:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077553.649654:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077553.649699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077553.649721:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077553.649734:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077553/real 1769077553] req@ffff935002a61c00 x1855008913486976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077553.649748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913486976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077553.649758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077553.649761:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077553.649765:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913486976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077553.649771:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913487104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077553.649792:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077553.649820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077553/real 1769077553] req@ffff935002a61f80 x1855008913487104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077553.649834:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913487104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077553.649843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077553.649846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077553.649850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913487104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077553.649856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913487232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077553.649877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077553.649890:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077553/real 1769077553] req@ffff935002a62680 x1855008913487232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077553.649902:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913487232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077553.649910:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077553.649913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077553.649916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913487232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077553.649923:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913487360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077553.649944:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077553.649955:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077553/real 1769077553] req@ffff935002a61180 x1855008913487360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077553.649974:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913487360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077553.649981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077553.649984:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077553.649987:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913487360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077553.649993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913487488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077553.650013:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077553.650024:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077553/real 1769077553] req@ffff935002a61880 x1855008913487488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077553.650036:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913487488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077553.650044:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077553.650071:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077553.650076:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913487488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077553.841116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077554.353140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077554.353141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077554.353141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077554.353169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077554.673164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077554.865190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077555.377156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077555.377156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077555.377157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077555.377213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077555.697143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077555.889117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077556.401147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077556.401147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077556.401147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077556.401181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077556.721222:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077556.913206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077557.425155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077557.425155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077557.425170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077557.425192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077557.745127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077557.937143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077558.449139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077558.449140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077558.449141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077558.449188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077558.769224:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077558.769230:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077558.769242:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077558.769250:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077558.769264:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3264 00000100:00080000:1.0:1769077558.769277:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077558.769286:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077558.769338:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913487616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077558.769370:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077558.769375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077558.769377:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077558.769392:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3264 00000100:00080000:1.0:1769077558.769398:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077558.769403:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077558.769411:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913487616:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077558.769417:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077558.769440:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913487744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077558.769453:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077558.769457:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077558.769459:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077558.769464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3264 00000100:00080000:1.0:1769077558.769469:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077558.769485:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077558.769488:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077558.769497:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077558.769515:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913487872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077558.769527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077558.769532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077558.769536:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077558/real 1769077558] req@ffff9350100cb800 x1855008913487616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077558.769536:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077558.769546:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3264 00000100:00080000:1.0:1769077558.769554:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077558.769561:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077558.769567:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077558.769572:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913487616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077558.769585:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913488000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077558.769587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077558.769598:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077558.769604:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913487616:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077558.769606:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077558.769613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077558.769615:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077558.769622:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3264 00000100:00080000:1.0:1769077558.769626:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077558.769632:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077558.769634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913487744:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769077558.769641:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077558.769660:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913488128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077558.769666:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077558.769680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077558.769681:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077558/real 1769077558] req@ffff9350100ca300 x1855008913487744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077558.769691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077558.769698:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913487744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077558.769706:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077558.769707:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077558.769712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077558.769733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913487744:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077558.769734:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3264 00000100:00100000:0.0:1769077558.769742:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913487872:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769077558.769743:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077558.769753:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077558.769760:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077558.769765:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077558.769780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077558/real 1769077558] req@ffff9350100c8380 x1855008913487872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077558.769780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913488256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077558.769807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913487872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077558.769814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077558.769826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077558.769832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077558.769832:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077558.769840:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077558.769844:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3264 00000100:00100000:0.0:1769077558.769846:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913487872:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769077558.769854:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077558.769855:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488000:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769077558.769865:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077558.769872:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769077558.769873:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077558.769882:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077558/real 1769077558] req@ffff9350100caa00 x1855008913488000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077558.769891:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913488384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077558.769894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913488000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077558.769925:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077558.769931:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077558.769934:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077558.769947:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913488128:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077558.769960:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077558.769967:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077558/real 1769077558] req@ffff9350100c9500 x1855008913488128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077558.769975:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913488128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077558.769981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077558.769983:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077558.769986:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913488128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077558.769989:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913488256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077558.769999:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077558.770005:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077558/real 1769077558] req@ffff9350100c9880 x1855008913488256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077558.770012:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913488256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077558.770018:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077558.770019:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077558.770021:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913488256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077558.770025:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913488384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077558.770034:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077558.770040:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077558/real 1769077558] req@ffff9350100c8e00 x1855008913488384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077558.770075:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913488384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077558.770082:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077558.770084:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077558.770087:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913488384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077558.961137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077559.473152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077559.473152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077559.473158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077559.473187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077559.793153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077559.985127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077560.497125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077560.497126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077560.497169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077560.497178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077560.817132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077561.009194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077561.521167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077561.521167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077561.521179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077561.521203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077561.841161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077562.033147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077562.545114:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077562.545118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077562.545122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077562.545151:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077562.865119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077563.057163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077563.569148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077563.569148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077563.569181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077563.569214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077563.889177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077563.889192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077563.889216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077563.889221:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077563.889231:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3269 00000100:00080000:0.0:1769077563.889243:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077563.889250:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077563.889290:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913488512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077563.889326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077563.889346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077563.889348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077563.889354:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3269 00000100:00080000:0.0:1769077563.889358:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077563.889364:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077563.889376:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077563.889408:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913488640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077563.889420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077563.889423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077563.889425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077563.889430:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913488512:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077563.889433:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3269 00000100:00080000:0.0:1769077563.889458:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077563.889464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077563.889472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077563.889487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913488768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077563.889497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077563.889501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077563.889502:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077563.889505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3269 00000100:00080000:0.0:1769077563.889509:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077563.889512:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077563.889516:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077563.889529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913488896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077563.889534:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077563.889546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077563.889550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077563.889552:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077563.889558:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3269 00000100:00080000:0.0:1769077563.889561:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077563.889565:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077563.889570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077563.889585:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913489024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077563.889593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077563.889596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077563.889597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077563] req@ffff9350103f9180 x1855008913488512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077563.889599:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077563.889610:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3269 00000100:00080000:0.0:1769077563.889617:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077563.889675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913488512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077563.889681:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077563.889696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077563.889702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077563.889706:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913488512:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769077563.889707:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077563.889729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913489152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077563.889733:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488640:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077563.889754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077563.889759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077563.889762:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077563.889764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077563.889773:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3269 00000100:00080000:0.0:1769077563.889780:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077563.889783:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077563] req@ffff9350103f9500 x1855008913488640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077563.889788:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077563.889797:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077563.889804:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913488640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077563.889815:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913489280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077563.889835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077563.889844:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077563.889849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077563.889856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077563.889873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077563.889881:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077563] req@ffff9350103fbb80 x1855008913488768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077563.889890:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913488768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077563.889896:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077563.889899:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077563.889902:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077563.889906:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077563.889916:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077563.889923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077563] req@ffff9350103f9c00 x1855008913488896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077563.889931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913488896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077563.889937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077563.889939:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077563.889942:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913488896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077563.889945:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489024:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077563.889956:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077563.889962:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077563] req@ffff9350103fb800 x1855008913489024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077563.889970:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913489024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077563.889976:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077563.889978:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077563.889980:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077563.889993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077563.890005:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077563.890011:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077563] req@ffff9350103f9f80 x1855008913489152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077563.890020:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913489152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077563.890026:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077563.890029:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077563.890031:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077563.890035:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077563.890045:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077563.890086:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077563] req@ffff9350103f8380 x1855008913489280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077563.890096:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913489280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077563.890102:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077563.890105:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077563.890107:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077564.081190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077564.593179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077564.593188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077564.593217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077564.593240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077564.913158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077565.105131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077565.617153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077565.617153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077565.617199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077565.617200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077565.937174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077566.129168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077566.641227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077566.641228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077566.641228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077566.641264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077566.961121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077567.153126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077567.665172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077567.665182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077567.665231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077567.665248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077567.985170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077568.177154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077568.689128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077568.689148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077568.689154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077568.689195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077569.009159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077569.009164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077569.009173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077569.009177:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077569.009185:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3274 00000100:00080000:2.0:1769077569.009213:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077569.009219:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077569.009254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913489408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077569.009295:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077569.009297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077569.009299:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077569.009306:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3274 00000100:00080000:2.0:1769077569.009309:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077569.009313:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077569.009339:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077569.009351:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913489408:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077569.009354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913489536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077569.009367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077569.009370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077569.009372:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077569.009376:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3274 00000100:00080000:2.0:1769077569.009379:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077569.009382:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077569.009385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077569.009395:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913489664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077569.009402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077569.009405:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077569.009407:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077569.009410:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3274 00000100:00080000:2.0:1769077569.009413:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077569.009415:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077569.009418:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077569.009423:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077569.009430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913489792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077569.009439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077569.009442:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077569.009444:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077569.009448:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3274 00000100:00080000:2.0:1769077569.009451:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077569.009454:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077569.009473:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077569.009482:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077569/real 1769077569] req@ffff935002a62300 x1855008913489408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077569.009485:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913489920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077569.009510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077569.009517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077569.009520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077569.009525:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3274 00000100:00080000:2.0:1769077569.009528:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077569.009531:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077569.009534:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077569.009535:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913489408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077569.009561:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769077569.009561:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913490048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077569.009568:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077569.009576:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913489408:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077569.009611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077569.009615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077569.009617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077569.009622:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3274 00000100:00080000:2.0:1769077569.009625:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077569.009629:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077569.009633:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077569.009638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913489536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077569.009648:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913490176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077569.009678:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077569.009694:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077569/real 1769077569] req@ffff935002a61500 x1855008913489536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077569.009721:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913489536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077569.009727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077569.009729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077569.009732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913489536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077569.009736:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913489664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077569.009751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077569.009757:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077569/real 1769077569] req@ffff935002a62a00 x1855008913489664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077569.009782:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913489664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077569.009794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077569.009797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077569.009799:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913489664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077569.009803:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913489792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077569.009815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077569.009822:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077569/real 1769077569] req@ffff935002a60000 x1855008913489792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077569.009830:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913489792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077569.009836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077569.009838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077569.009840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913489792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077569.009845:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489920:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077569.009865:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077569.009878:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077569/real 1769077569] req@ffff935002a61880 x1855008913489920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077569.009893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913489920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077569.009904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077569.009908:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077569.009913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913489920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077569.009920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077569.009941:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077569.009950:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077569/real 1769077569] req@ffff935002a61180 x1855008913490048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077569.009976:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913490048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077569.009981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077569.009983:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077569.009985:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077569.009999:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077569.010011:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077569.010018:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077569/real 1769077569] req@ffff935002a62680 x1855008913490176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077569.010025:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913490176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077569.010030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077569.010032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077569.010034:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077569.201136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077569.713169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077569.713169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077569.713178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077569.713253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077570.033184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077570.225179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077570.737134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077570.737134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077570.737144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077570.737209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077571.057223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077571.249120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077571.761161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077571.761161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077571.761163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077571.761216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077572.081165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077572.273147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077572.785121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077572.785121:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077572.785131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077572.785193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077573.105197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077573.297106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077573.809158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077573.809191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077573.809204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077573.809218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077574.129165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077574.129178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077574.129186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077574.129190:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077574.129199:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3279 00000100:00080000:0.0:1769077574.129212:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077574.129219:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077574.129258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913490304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077574.129285:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077574.129288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077574.129289:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077574.129293:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3279 00000100:00080000:0.0:1769077574.129296:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077574.129299:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077574.129308:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077574.129318:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913490432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077574.129327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077574.129330:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913490304:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077574.129331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077574.129335:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077574.129341:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3279 00000100:00080000:0.0:1769077574.129343:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077574.129346:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077574.129350:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077574.129361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913490560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077574.129367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077574.129370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077574.129372:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077574.129374:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3279 00000100:00080000:0.0:1769077574.129377:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077574.129379:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077574.129382:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077574.129389:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913490688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077574.129396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077574.129400:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077574.129401:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077574.129405:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077574.129409:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3279 00000100:00080000:0.0:1769077574.129411:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077574.129414:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077574.129418:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077574.129426:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913490816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077574.129432:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077574.129436:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077574.129437:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077574.129441:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077574/real 1769077574] req@ffff9350103faa00 x1855008913490304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077574.129444:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3279 00000100:00080000:0.0:1769077574.129450:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077574.129456:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077574.129482:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913490304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077574.129483:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077574.129495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077574.129500:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913490944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077574.129501:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077574.129507:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913490304:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077574.129522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077574.129526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077574.129529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077574.129533:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913490432:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077574.129535:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3279 00000100:00080000:0.0:1769077574.129544:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077574.129549:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077574.129555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077574.129559:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077574.129572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077574/real 1769077574] req@ffff9350103fb100 x1855008913490432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077574.129577:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913491072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077574.129588:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913490432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077574.129600:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077574.129603:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077574.129607:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913490432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077574.129613:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913490560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077574.129626:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077574.129632:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077574/real 1769077574] req@ffff9350103f8700 x1855008913490560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077574.129640:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913490560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077574.129646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077574.129648:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077574.129650:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913490560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077574.129654:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913490688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077574.129676:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077574.129683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077574/real 1769077574] req@ffff9350103fad80 x1855008913490688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077574.129690:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913490688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077574.129696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077574.129698:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077574.129701:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913490688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077574.129704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490816:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077574.129714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077574.129720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077574/real 1769077574] req@ffff9350103fb480 x1855008913490816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077574.129728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913490816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077574.129733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077574.129735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077574.129737:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077574.129741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077574.129750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077574.129756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077574/real 1769077574] req@ffff9350103f8380 x1855008913490944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077574.129764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913490944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077574.129769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077574.129771:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077574.129773:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913490944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077574.129785:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077574.129795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077574.129801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077574/real 1769077574] req@ffff9350103f9f80 x1855008913491072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077574.129808:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913491072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077574.129814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077574.129816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077574.129818:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077574.321211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077574.833187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077574.833196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077574.833199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077574.833248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077575.153170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077575.345144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077575.857165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077575.857167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077575.857174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077575.857249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077576.177142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077576.369183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077576.881188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077576.881230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077576.881232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077576.881252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077577.201145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077577.393132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077577.905150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077577.905159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077577.905196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077577.905229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077578.225176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077578.417181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077578.929162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077578.929162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077578.929227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077578.929300:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:1.0:1769077579.249130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077579.249140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077579.249143:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077579.249151:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3284 00000100:00080000:1.0:1769077579.249161:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077579.249167:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077579.249202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913491200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077579.249220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077579.249225:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077579.249230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077579.249231:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077579.249243:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3284 00000100:00080000:1.0:1769077579.249246:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077579.249257:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077579.249268:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077579.249269:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913491200:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077579.249282:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913491328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077579.249291:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077579.249295:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077579.249296:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077579.249300:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3284 00000100:00080000:1.0:1769077579.249302:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077579.249305:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077579.249309:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077579.249318:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913491456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077579.249325:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077579.249327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077579.249328:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077579.249330:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3284 00000100:00080000:1.0:1769077579.249332:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077579.249335:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077579.249338:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077579.249340:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077579.249348:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913491584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077579.249356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077579.249359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077579.249360:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077579.249363:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3284 00000100:00080000:1.0:1769077579.249366:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077579.249368:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077579.249386:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077579.249396:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913491712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077579.249404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077579.249407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077579.249410:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077579/real 1769077579] req@ffff9350100ca680 x1855008913491200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077579.249410:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077579.249417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3284 00000100:00080000:1.0:1769077579.249422:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077579.249429:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077579.249434:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077579.249443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913491840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077579.249446:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913491200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077579.249454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077579.249462:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077579.249464:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077579.249465:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077579.249468:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3284 00000100:00080000:1.0:1769077579.249474:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077579.249478:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077579.249480:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077579.249486:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077579.249489:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913491200:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077579.249496:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913491968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077579.249523:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913491328:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077579.249545:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077579.249555:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077579/real 1769077579] req@ffff9350100cad80 x1855008913491328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077579.249564:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913491328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077579.249571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077579.249574:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077579.249576:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913491328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077579.249580:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913491456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077579.249593:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077579.249599:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077579/real 1769077579] req@ffff9350100c9c00 x1855008913491456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077579.249612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913491456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077579.249627:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077579.249631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077579.249634:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913491456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077579.249641:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913491584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077579.249663:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077579.249672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077579/real 1769077579] req@ffff9350100cb100 x1855008913491584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077579.249681:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913491584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077579.249687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077579.249689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077579.249692:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913491584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077579.249695:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491712:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077579.249707:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077579.249714:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077579/real 1769077579] req@ffff9350100c9180 x1855008913491712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077579.249721:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913491712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077579.249726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077579.249728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077579.249731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077579.249734:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077579.249744:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077579.249749:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077579/real 1769077579] req@ffff9350100c8a80 x1855008913491840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077579.249756:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913491840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077579.249760:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077579.249763:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077579.249765:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077579.249768:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077579.249777:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077579.249782:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077579/real 1769077579] req@ffff9350100c8000 x1855008913491968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077579.249788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913491968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077579.249793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077579.249795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077579.249797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913491968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077579.441161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077579.953172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077579.953172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077579.953191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077579.953244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077580.273251:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077580.465183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077580.977179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077580.977179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077580.977220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077580.977280:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077581.297182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077581.489163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077582.001136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077582.001137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077582.001144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077582.001176:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077582.321162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077582.513174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077583.025159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077583.025159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077583.025159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077583.025183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077583.345123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077583.537155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077584.049145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077584.049145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077584.049164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077584.049206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077584.369158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077584.369158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077584.369168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077584.369172:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077584.369181:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3290 00000100:00080000:0.0:1769077584.369192:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077584.369198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077584.369232:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913492096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077584.369258:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077584.369261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077584.369262:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077584.369266:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3290 00000100:00080000:0.0:1769077584.369268:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077584.369271:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077584.369279:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077584.369306:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913492224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077584.369316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077584.369321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077584.369321:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913492096:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077584.369324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077584.369332:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3290 00000100:00080000:0.0:1769077584.369335:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077584.369340:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077584.369345:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077584.369355:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913492352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077584.369362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077584.369365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077584.369367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077584.369369:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3290 00000100:00080000:0.0:1769077584.369371:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077584.369373:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077584.369376:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077584.369390:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913492480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077584.369397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077584.369400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077584.369402:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077584.369404:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3290 00000100:00080000:0.0:1769077584.369407:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077584.369410:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077584.369413:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077584.369425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913492608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077584.369434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077584.369434:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913492224:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077584.369443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077584.369446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077584.369450:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3290 00000100:00080000:0.0:1769077584.369453:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077584.369457:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077584.369461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077584.369471:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913492352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077584.369473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913492736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077584.369484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077584.369489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077584.369491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077584.369493:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913492480:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077584.369500:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3290 00000100:00080000:0.0:1769077584.369507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077584.369514:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077584.369519:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913492608:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769077584.369520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077584.369540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913492864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077584.369547:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913492736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077584.369570:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913492864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077584.561121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077585.073147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077585.073148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077585.073148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077585.073181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077585.393144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077585.585171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077586.097124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077586.097124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077586.097136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077586.097172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077586.417156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077586.609128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077587.121154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077587.121154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077587.121154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077587.121186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077587.441120:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077587.441363:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077587.441401:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077587.441418:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077587.441468:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077587.441479:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077587.441487:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077587.441496:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077587.441505:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077587.441513:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077587.441593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077584/real 1769077587] req@ffff9350103f8380 x1855008913492864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077587.441621:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913492864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077587.441633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077587.441639:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077587.441643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913492864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077587.441651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077584/real 1769077587] req@ffff9350103f9180 x1855008913492608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077587.441663:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913492608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077587.441671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077587.441674:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077587.441678:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913492608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077587.441685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077584/real 1769077587] req@ffff9350103f9f80 x1855008913492736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077587.441696:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913492736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077587.441704:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077587.441708:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077587.441711:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913492736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077587.441717:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077584/real 1769077587] req@ffff9350103f9c00 x1855008913492224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077587.441729:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913492224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077587.441738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077587.441741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077587.441744:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913492224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077587.441751:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077584/real 1769077587] req@ffff9350103fbb80 x1855008913492352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077587.441763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913492352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077587.441770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077587.441774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077587.441777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913492352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077587.441784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077584/real 1769077587] req@ffff9350103f9500 x1855008913492480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077587.441794:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913492480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077587.441802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077587.441806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077587.441808:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913492480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077587.441815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077584/real 1769077587] req@ffff9350103fb800 x1855008913492096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077587.441825:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913492096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077587.441833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077587.441836:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077587.441840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913492096:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077587.633159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077588.145128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077588.145132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077588.145143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077588.145188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077588.465127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077588.657124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077589.169142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077589.169142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077589.169159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077589.169187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077589.489174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077589.489192:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077589.489196:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077589.489219:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3295 00000100:00100000:2.0:1769077589.489221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077589.489230:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077589.489238:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077589.489272:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913492992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077589.489301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077589.489305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077589.489307:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077589.489312:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3295 00000100:00080000:1.0:1769077589.489317:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077589.489322:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077589.489332:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077589.489346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913492992:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077589.489350:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913493120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077589.489365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077589.489369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077589.489371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077589.489376:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3295 00000100:00080000:1.0:1769077589.489381:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077589.489386:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077589.489391:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077589.489404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913493248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077589.489414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077589.489418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077589.489419:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077589.489424:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3295 00000800:00000100:2.0:1769077589.489430:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077589.489432:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077589.489440:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077589.489446:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077589.489459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913493376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077589.489472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077589.489475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077589.489477:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077589.489483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3295 00000100:00100000:2.0:1769077589.489485:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077589/real 1769077589] req@ffff9350100c9f80 x1855008913492992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077589.489488:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077589.489497:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077589.489507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077589.489522:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913493504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077589.489533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077589.489535:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913492992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077589.489541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077589.489545:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077589.489555:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3295 00000100:00080000:1.0:1769077589.489562:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077589.489564:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077589.489568:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077589.489575:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077589.489582:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913492992:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077589.489583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077589.489601:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913493632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077589.489612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077589.489615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077589.489617:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077589.489621:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3295 00000100:00080000:1.0:1769077589.489627:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077589.489631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913493120:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077589.489635:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077589.489645:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077589.489661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913493760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077589.489674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077589.489695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077589/real 1769077589] req@ffff9350100c8000 x1855008913493120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077589.489711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913493120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077589.489722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077589.489727:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077589.489731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913493120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077589.489739:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913493248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077589.489763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077589.489785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077589/real 1769077589] req@ffff9350100c8a80 x1855008913493248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077589.489799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913493248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077589.489809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077589.489813:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077589.489817:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913493248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077589.489824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913493376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077589.489848:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077589.489862:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077589/real 1769077589] req@ffff9350100c9180 x1855008913493376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077589.489875:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913493376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077589.489883:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077589.489887:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077589.489891:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913493376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077589.489897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913493504:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077589.489920:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077589.489933:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077589/real 1769077589] req@ffff9350100cb100 x1855008913493504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077589.489947:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913493504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077589.489969:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077589.489973:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077589.489977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913493504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077589.489985:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913493632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077589.490011:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077589.490027:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077589/real 1769077589] req@ffff9350100c9c00 x1855008913493632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077589.490041:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913493632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077589.490095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077589.490100:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077589.490103:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913493632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077589.490125:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913493760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077589.490148:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077589.490161:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077589/real 1769077589] req@ffff9350100cad80 x1855008913493760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077589.490174:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913493760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077589.490184:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077589.490188:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077589.490192:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913493760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077589.681150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077590.193120:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077590.193124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077590.193131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077590.193152:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077590.513146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077590.705108:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077591.217124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077591.217124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077591.217127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077591.217162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077591.537128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077591.729122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077592.241115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077592.241125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077592.241144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077592.241189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077592.561123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077592.753160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077593.265164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077593.265164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077593.265164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077593.265186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077593.585189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077593.777164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077594.289180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077594.289183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077594.289200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077594.289215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077594.609175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077594.609181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077594.609190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077594.609195:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077594.609204:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3300 00000100:00080000:0.0:1769077594.609227:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077594.609235:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077594.609277:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913493888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077594.609308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077594.609312:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077594.609314:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077594.609318:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3300 00000100:00080000:0.0:1769077594.609322:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077594.609327:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077594.609338:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077594.609346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913493888:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077594.609357:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913494016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077594.609369:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077594.609373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077594.609375:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077594.609379:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3300 00000100:00080000:0.0:1769077594.609383:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077594.609387:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077594.609393:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077594.609409:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077594.609410:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913494144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077594.609433:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077594.609436:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077594.609438:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077594.609443:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3300 00000100:00100000:2.0:1769077594.609447:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077594/real 1769077594] req@ffff9350103fb480 x1855008913493888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077594.609450:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077594.609459:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077594.609466:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077594.609469:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913493888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077594.609480:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077594.609484:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913494272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077594.609486:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077594.609492:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913493888:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077594.609507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077594.609511:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077594.609513:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077594.609520:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3300 00000100:00100000:2.0:1769077594.609520:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494016:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077594.609528:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077594.609533:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077594.609539:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077594.609546:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077594.609553:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913494400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077594.609560:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077594/real 1769077594] req@ffff9350103fad80 x1855008913494016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077594.609567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077594.609573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077594.609575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077594.609578:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913494016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077594.609582:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3300 00000100:00080000:0.0:1769077594.609588:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077594.609595:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077594.609601:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077594.609605:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077594.609611:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077594.609614:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913494528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077594.609618:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077594.609631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494144:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077594.609632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077594.609638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077594.609639:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077594.609643:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3300 00000100:00080000:0.0:1769077594.609646:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077594.609649:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077594.609653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:0.0:1769077594.609654:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077594.609664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077594/real 1769077594] req@ffff9350103f8700 x1855008913494144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077594.609668:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913494656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077594.609688:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913494144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077594.609696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077594.609700:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077594.609703:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077594.609707:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077594.609719:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077594.609724:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077594/real 1769077594] req@ffff9350103fb100 x1855008913494272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077594.609732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913494272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077594.609737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077594.609739:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077594.609742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077594.609752:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913494400:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077594.609762:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077594.609768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077594/real 1769077594] req@ffff9350103faa00 x1855008913494400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077594.609776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913494400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077594.609782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077594.609784:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077594.609786:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913494400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077594.609789:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913494528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077594.609798:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077594.609803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077594/real 1769077594] req@ffff9350103fb480 x1855008913494528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077594.609810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913494528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077594.609816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077594.609818:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077594.609821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913494528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077594.609826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913494656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077594.609843:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077594.609854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077594/real 1769077594] req@ffff9350103fb800 x1855008913494656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077594.609866:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913494656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077594.609874:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077594.609877:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077594.609880:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913494656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077594.801145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077595.313157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077595.313157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077595.313157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077595.313195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077595.633177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077595.825139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077596.337166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077596.337166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077596.337166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077596.337209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077596.657158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077596.849138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077597.361165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077597.361165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077597.361165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077597.361213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077597.681183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077597.873152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077598.385165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077598.385165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077598.385213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077598.385221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077598.705196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077598.897169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077599.409149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077599.409168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077599.409168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077599.409261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077599.729166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077599.729193:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077599.729200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077599.729204:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077599.729211:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3305 00000100:00080000:3.0:1769077599.729222:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077599.729227:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077599.729260:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913494784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077599.729286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077599.729288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077599.729290:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077599.729293:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3305 00000100:00080000:3.0:1769077599.729295:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077599.729298:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077599.729306:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077599.729315:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913494912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077599.729322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077599.729325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077599.729327:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077599.729329:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3305 00000100:00080000:3.0:1769077599.729331:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077599.729334:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077599.729337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077599.729343:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913494784:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077599.729353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913495040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077599.729363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077599.729366:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077599.729367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077599.729370:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3305 00000100:00080000:3.0:1769077599.729373:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077599.729375:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077599.729378:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077599.729387:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913495168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077599.729393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077599.729397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077599.729398:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077599.729401:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3305 00000100:00080000:3.0:1769077599.729405:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077599.729408:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077599.729412:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077599.729418:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077599.729428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913495296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077599.729435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077599.729438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077599.729439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077599.729442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3305 00000100:00080000:3.0:1769077599.729446:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077599.729449:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077599/real 1769077599] req@ffff935002a7c000 x1855008913494784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077599.729451:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077599.729458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077599.729468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913495424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077599.729475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077599.729477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077599.729479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077599.729481:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913494784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077599.729484:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3305 00000100:00080000:3.0:1769077599.729490:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077599.729496:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077599.729500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769077599.729500:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077599.729509:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077599.729514:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913495552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077599.729517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913494784:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077599.729529:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494912:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077599.729563:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077599.729577:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077599/real 1769077599] req@ffff935002a7f800 x1855008913494912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077599.729598:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913494912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077599.729606:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077599.729616:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077599.729618:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913494912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077599.729640:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077599.729659:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077599.729671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077599/real 1769077599] req@ffff935002a7dc00 x1855008913495040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077599.729685:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913495040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077599.729694:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077599.729698:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077599.729702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077599.729718:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077599.729734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077599.729746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077599/real 1769077599] req@ffff935002a7d180 x1855008913495168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077599.729759:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913495168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077599.729775:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077599.729779:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077599.729783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077599.729790:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913495296:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077599.729806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077599.729821:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077599/real 1769077599] req@ffff935002a7df80 x1855008913495296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077599.729829:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913495296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077599.729835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077599.729838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077599.729840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913495296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077599.729844:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913495424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077599.729855:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077599.729861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077599/real 1769077599] req@ffff935002a7fb80 x1855008913495424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077599.729869:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913495424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077599.729875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077599.729877:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077599.729880:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913495424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077599.729884:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913495552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077599.729893:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077599.729898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077599/real 1769077599] req@ffff935002a7f100 x1855008913495552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077599.729906:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913495552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077599.729911:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077599.729913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077599.729915:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913495552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077599.921161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077600.433155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077600.433155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077600.433186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077600.433220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077600.753161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077600.945160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077601.457120:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077601.457128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077601.457128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077601.457162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077601.777166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077601.969144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077602.481160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077602.481160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077602.481160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077602.481194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077602.801139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077602.993157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077603.505150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077603.505150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077603.505150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077603.505183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077603.825214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077604.017187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077604.529164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077604.529164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077604.529192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077604.529221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077604.849138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077604.849143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077604.849150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077604.849155:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077604.849163:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3310 00000100:00080000:3.0:1769077604.849174:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077604.849180:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077604.849212:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913495680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077604.849236:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077604.849240:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077604.849242:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077604.849247:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3310 00000100:00080000:3.0:1769077604.849251:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077604.849256:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077604.849266:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077604.849276:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913495680:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077604.849281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913495808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077604.849291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077604.849294:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077604.849296:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077604.849300:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3310 00000100:00080000:3.0:1769077604.849303:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077604.849307:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077604.849312:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077604.849321:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913495936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077604.849330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077604.849333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077604.849335:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077604.849338:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3310 00000800:00000100:2.0:1769077604.849341:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077604.849342:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077604.849345:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077604.849349:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077604.849358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913496064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077604.849367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077604.849371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077604.849373:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077604.849377:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3310 00000100:00100000:2.0:1769077604.849378:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077604/real 1769077604] req@ffff935002a7c380 x1855008913495680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077604.849381:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077604.849385:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077604.849389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077604.849396:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913495680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077604.849401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913496192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077604.849408:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077604.849410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077604.849412:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077604.849414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077604.849416:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077604.849417:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913495680:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077604.849419:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3310 00000100:00080000:3.0:1769077604.849422:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077604.849425:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077604.849430:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077604.849440:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077604.849440:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913496320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077604.849449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077604.849453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077604.849455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077604.849459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3310 00000800:00000100:2.0:1769077604.849461:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077604.849463:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077604.849466:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077604.849471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077604.849475:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077604/real 1769077604] req@ffff935002a7ca80 x1855008913495808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077604.849481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913496448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077604.849489:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913495808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.849498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.849502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077604.849506:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077604.849512:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077604.849533:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077604.849551:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077604/real 1769077604] req@ffff935002a7e300 x1855008913495936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.849560:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913495936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.849566:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.849569:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077604.849571:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913495936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077604.849575:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077604.849587:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077604.849593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077604/real 1769077604] req@ffff935002a7c700 x1855008913496064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.849601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913496064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.849607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.849609:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077604.849611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077604.849614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913496192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077604.849625:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077604.849644:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077604/real 1769077604] req@ffff935002a7f100 x1855008913496192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.849657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913496192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.849666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.849670:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077604.849673:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913496192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077604.849700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913496320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077604.849717:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077604.849726:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077604/real 1769077604] req@ffff935002a7fb80 x1855008913496320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.849734:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913496320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.849740:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.849742:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077604.849745:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913496320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077604.849749:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913496448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077604.849759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077604.849764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077604/real 1769077604] req@ffff935002a7df80 x1855008913496448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.849771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913496448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.849777:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.849778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077604.849780:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913496448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077605.041188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077605.553129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077605.553133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077605.553135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077605.553164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077605.873151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077606.065195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077606.577160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077606.577160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077606.577160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077606.577200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077606.897158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077607.089185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077607.601132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077607.601132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077607.601183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077607.601243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077607.921158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077608.113120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077608.625151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077608.625151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077608.625151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077608.625329:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077608.945166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077609.137164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077609.649130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077609.649130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077609.649140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077609.649168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077609.969135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077609.969138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077609.969146:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077609.969151:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077609.969159:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3315 00000100:00080000:3.0:1769077609.969172:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077609.969179:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077609.969211:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913496576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077609.969234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077609.969237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077609.969238:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077609.969241:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3315 00000100:00080000:3.0:1769077609.969244:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077609.969247:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077609.969256:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077609.969267:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913496576:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077609.969270:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913496704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077609.969279:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077609.969282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077609.969283:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077609.969286:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3315 00000100:00080000:3.0:1769077609.969288:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077609.969290:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077609.969294:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077609.969306:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913496832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077609.969313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077609.969316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077609.969317:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077609.969319:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3315 00000100:00080000:3.0:1769077609.969324:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077609.969327:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077609.969331:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077609.969335:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077609.969343:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913496960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077609.969352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077609.969354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077609.969356:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077609.969360:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3315 00000100:00080000:3.0:1769077609.969365:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077609.969369:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077609.969375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077609.969383:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff935002a7d180 x1855008913496576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077609.969393:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913497088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077609.969404:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913496576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077609.969410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077609.969416:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077609.969417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077609.969423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077609.969428:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077609.969432:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3315 00000100:00080000:3.0:1769077609.969438:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077609.969442:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077609.969448:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913496576:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077609.969450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077609.969465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913497216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077609.969473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077609.969475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077609.969477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077609.969480:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3315 00000100:00080000:3.0:1769077609.969483:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077609.969486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496704:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077609.969489:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077609.969496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077609.969507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913497344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077609.969520:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077609.969533:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff935002a7dc00 x1855008913496704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.969543:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913496704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.969550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.969553:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077609.969556:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077609.969561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077609.969572:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077609.969579:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff935002a7f800 x1855008913496832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.969587:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913496832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.969592:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.969594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077609.969596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077609.969600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077609.969610:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077609.969615:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff935002a7c000 x1855008913496960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.969623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913496960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.969629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.969631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077609.969633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913496960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077609.969639:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497088:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077609.969658:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077609.969669:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff935002a7df80 x1855008913497088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.969683:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913497088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.969692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.969696:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077609.969699:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077609.969705:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077609.969720:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077609.969728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff935002a7fb80 x1855008913497216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.969736:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913497216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.969751:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.969755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077609.969759:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077609.969781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077609.969802:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077609.969811:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff935002a7f100 x1855008913497344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.969819:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913497344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.969825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.969827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077609.969829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077610.161129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077610.673149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077610.673149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077610.673163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077610.673209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077610.993208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077611.185122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077611.697134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077611.697137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077611.697146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077611.697172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077612.017137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077612.209148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077612.721165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077612.721165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077612.721165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077612.721202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077613.041175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077613.233122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077613.745164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077613.745164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077613.745164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077613.745192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077614.065131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077614.257163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077614.769146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077614.769146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077614.769147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077614.769174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077615.089204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077615.089216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077615.089220:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077615.089231:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3320 00000100:00080000:1.0:1769077615.089243:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077615.089251:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077615.089292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913497472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077615.089324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077615.089329:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077615.089331:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077615.089336:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3320 00000100:00080000:1.0:1769077615.089340:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077615.089345:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077615.089355:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077615.089372:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913497600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077615.089383:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913497472:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077615.089389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077615.089395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077615.089397:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077615.089403:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3320 00000100:00080000:1.0:1769077615.089408:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077615.089412:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077615.089419:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077615.089431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913497728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077615.089442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077615.089448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769077615.089448:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077615.089452:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077615.089459:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3320 00000100:00080000:1.0:1769077615.089464:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077615.089468:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077615.089472:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077615.089485:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913497856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077615.089500:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077615/real 1769077615] req@ffff9350100ca680 x1855008913497472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077615.089502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077615.089512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077615.089516:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077615.089524:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3320 00000100:00100000:2.0:1769077615.089527:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913497472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077615.089530:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077615.089540:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077615.089551:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077615.089553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077615.089560:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077615.089564:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913497472:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077615.089570:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913497984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077615.089581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077615.089586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077615.089588:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913497600:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077615.089590:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077615.089597:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3320 00000100:00080000:1.0:1769077615.089603:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077615.089608:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077615.089612:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769077615.089617:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077615.089625:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077615/real 1769077615] req@ffff9350100c8e00 x1855008913497600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077615.089635:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913498112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077615.089638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913497600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077615.089664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077615.089669:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077615.089670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077615.089672:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913497600:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077615.089680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077615.089684:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913497728:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769077615.089686:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077615.089693:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3320 00000100:00080000:1.0:1769077615.089699:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077615.089703:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077615.089708:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077615.089715:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077615/real 1769077615] req@ffff9350100c9880 x1855008913497728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077615.089716:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077615.089730:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913497728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077615.089738:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913498240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077615.089739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077615.089746:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077615.089754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913497728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077615.089766:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913497856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077615.089805:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077615.089819:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769077615.089821:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077615/real 1769077615] req@ffff9350100c9500 x1855008913497856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077615.089836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913497856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077615.089843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077615.089846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077615.089849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913497856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077615.089855:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497984:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077615.089874:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077615.089885:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077615/real 1769077615] req@ffff9350100caa00 x1855008913497984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077615.089897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913497984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077615.089907:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077615.089910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077615.089915:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913497984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077615.089933:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913498112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077615.089950:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077615.089957:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077615/real 1769077615] req@ffff9350100c8380 x1855008913498112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077615.089966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913498112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077615.089972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077615.089974:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077615.089976:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913498112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077615.089979:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913498240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077615.089989:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077615.089995:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077615/real 1769077615] req@ffff9350100ca300 x1855008913498240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077615.090002:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913498240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077615.090007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077615.090009:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077615.090011:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913498240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077615.281226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077615.793159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077615.793185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077615.793188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077615.793192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077616.113172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077616.305171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077616.817134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077616.817134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077616.817158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077616.817170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077617.137162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077617.329152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077617.841138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077617.841140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077617.841171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077617.841253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077618.161147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077618.353115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077618.865186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077618.865186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077618.865186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077618.865212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077619.185169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077619.377176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077619.889172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077619.889172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077619.889178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077619.889207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077620.209186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077620.209194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077620.209202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077620.209207:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077620.209216:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3325 00000100:00080000:0.0:1769077620.209227:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077620.209234:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077620.209277:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913498368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077620.209301:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077620.209305:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077620.209306:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077620.209309:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3325 00000100:00080000:0.0:1769077620.209312:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077620.209314:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077620.209325:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077620.209334:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913498496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077620.209343:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077620.209347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077620.209348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913498368:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077620.209349:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077620.209357:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3325 00000100:00080000:0.0:1769077620.209361:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077620.209363:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077620.209369:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077620.209381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913498624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077620.209392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077620.209396:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077620.209398:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077620.209402:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3325 00000100:00080000:0.0:1769077620.209407:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077620.209412:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077620.209418:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077620.209430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913498752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077620.209438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077620.209441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077620.209442:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077620.209445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3325 00000100:00080000:0.0:1769077620.209450:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077620.209450:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077620.209457:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077620.209462:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077620.209471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913498880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077620.209478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077620.209480:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077620.209481:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077620.209484:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3325 00000100:00080000:0.0:1769077620.209486:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077620.209489:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077620.209492:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077620.209504:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077620/real 1769077620] req@ffff9350103f9500 x1855008913498368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077620.209505:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913499008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077620.209534:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077620.209539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077620.209543:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077620.209549:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913498368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077620.209550:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3325 00000100:00080000:0.0:1769077620.209557:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077620.209566:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077620.209572:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769077620.209574:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077620.209583:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077620.209588:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913498368:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077620.209592:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913499136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077620.209616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913498496:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077620.209637:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077620.209648:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077620/real 1769077620] req@ffff9350103fbb80 x1855008913498496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077620.209657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913498496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077620.209663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077620.209666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077620.209668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913498496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077620.209673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913498624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077620.209702:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077620.209711:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077620/real 1769077620] req@ffff9350103f9c00 x1855008913498624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077620.209720:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913498624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077620.209729:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077620.209732:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077620.209734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913498624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077620.209738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913498752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077620.209750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077620.209756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077620/real 1769077620] req@ffff9350103f9f80 x1855008913498752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077620.209764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913498752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077620.209769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077620.209771:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077620.209772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913498752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077620.209776:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913498880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077620.209786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077620.209792:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077620/real 1769077620] req@ffff9350103f9180 x1855008913498880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077620.209800:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913498880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077620.209809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077620.209812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077620.209816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913498880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077620.209823:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077620.209842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077620.209853:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077620/real 1769077620] req@ffff9350103f8380 x1855008913499008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077620.209866:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913499008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077620.209875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077620.209878:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077620.209882:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077620.209887:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077620.209902:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077620.209910:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077620/real 1769077620] req@ffff9350103fb800 x1855008913499136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077620.209918:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913499136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077620.209923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077620.209925:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077620.209927:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077620.401135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077620.913150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077620.913150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077620.913196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077620.913215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077621.233145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077621.425171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077621.937188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077621.937188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077621.937205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077621.937226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077622.257219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077622.449157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077622.961163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077622.961164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077622.961176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077622.961258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077623.281154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077623.473172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077623.985185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077623.985191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077623.985237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077623.985296:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077624.305199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077624.497163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077625.009177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077625.009177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077625.009191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077625.009247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077625.329152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077625.329235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077625.329245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077625.329250:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077625.329260:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3331 00000100:00080000:2.0:1769077625.329272:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077625.329278:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077625.329322:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913499264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077625.329359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077625.329362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077625.329364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077625.329367:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3331 00000100:00080000:2.0:1769077625.329369:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077625.329372:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077625.329382:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077625.329391:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913499392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077625.329398:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077625.329400:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077625.329401:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077625.329404:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3331 00000100:00080000:2.0:1769077625.329406:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077625.329409:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077625.329412:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077625.329419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913499520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077625.329430:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913499264:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077625.329433:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077625.329438:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077625.329440:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077625.329443:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3331 00000100:00080000:2.0:1769077625.329445:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077625.329448:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077625.329451:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077625.329459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913499648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077625.329464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077625.329467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077625.329468:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077625.329471:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3331 00000100:00080000:2.0:1769077625.329473:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077625.329476:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077625.329479:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077625.329486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913499776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077625.329492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077625.329494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077625.329496:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077625.329498:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3331 00000100:00080000:2.0:1769077625.329502:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077625.329504:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077625.329507:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077625.329512:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077625.329519:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913499904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077625.329525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077625.329528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077625.329529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077625.329532:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3331 00000100:00080000:2.0:1769077625.329543:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077625.329545:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077625.329548:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077625.329556:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913500032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077625.329564:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077625/real 1769077625] req@ffff935002a61f80 x1855008913499264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077625.329581:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913499264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077625.329590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077625.329594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077625.329611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913499264:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077625.329618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913499392:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077625.329634:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077625.329644:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077625/real 1769077625] req@ffff935002a61c00 x1855008913499392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077625.329652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913499392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077625.329659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077625.329661:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077625.329664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913499392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077625.329688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913499520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077625.329700:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077625.329706:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077625/real 1769077625] req@ffff935002a63100 x1855008913499520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077625.329714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913499520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077625.329719:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077625.329721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077625.329722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913499520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077625.329726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913499648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077625.329735:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077625.329741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077625/real 1769077625] req@ffff935002a60a80 x1855008913499648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077625.329748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913499648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077625.329754:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077625.329756:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077625.329758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913499648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077625.329761:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499776:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077625.329771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077625.329776:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077625/real 1769077625] req@ffff935002a62680 x1855008913499776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077625.329783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913499776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077625.329788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077625.329790:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077625.329792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077625.329796:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077625.329804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077625.329810:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077625/real 1769077625] req@ffff935002a61180 x1855008913499904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077625.329816:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913499904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077625.329821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077625.329823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077625.329826:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913499904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077625.329829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077625.329838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077625.329844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077625/real 1769077625] req@ffff935002a61880 x1855008913500032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077625.329851:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913500032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077625.329856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077625.329858:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077625.329860:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077625.521171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077626.033176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077626.033176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077626.033216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077626.033221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077626.353176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077626.545217:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077627.057153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077627.057207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077627.057232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077627.057235:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077627.377151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077627.569112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077628.081204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077628.081204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077628.081246:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077628.081294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077628.401195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077628.593181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077629.105190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077629.105190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077629.105241:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077629.105259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077629.425177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077629.617148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077630.129180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077630.129180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077630.129180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077630.129230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077630.449158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077630.449160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077630.449168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077630.449173:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077630.449184:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3336 00000100:00080000:2.0:1769077630.449196:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077630.449212:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077630.449254:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913500160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077630.449278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077630.449282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077630.449283:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077630.449286:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3336 00000100:00080000:2.0:1769077630.449289:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077630.449292:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077630.449302:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077630.449311:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913500288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077630.449320:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077630.449323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077630.449324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077630.449329:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3336 00000100:00100000:0.0:1769077630.449331:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913500160:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077630.449331:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077630.449338:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077630.449345:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077630.449356:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913500416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077630.449367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077630.449370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077630.449372:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077630.449377:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3336 00000100:00080000:2.0:1769077630.449381:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077630.449385:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077630.449392:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077630.449404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913500544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077630.449415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077630.449418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077630.449419:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077630.449425:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077630.449427:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3336 00000100:00080000:2.0:1769077630.449432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077630.449435:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077630.449440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077630.449451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913500672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077630.449458:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077630.449460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077630.449462:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077630.449465:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3336 00000100:00080000:2.0:1769077630.449467:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077630.449470:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077630.449473:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077630.449476:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077630/real 1769077630] req@ffff935002a60000 x1855008913500160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077630.449483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913500800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077630.449493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077630.449496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077630.449498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077630.449502:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913500160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077630.449504:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3336 00000100:00080000:2.0:1769077630.449510:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077630.449531:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077630.449532:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769077630.449535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077630.449539:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077630.449545:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913500160:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077630.449549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913500928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077630.449577:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913500288:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077630.449597:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077630.449608:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077630/real 1769077630] req@ffff935002a62a00 x1855008913500288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077630.449617:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913500288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077630.449624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077630.449627:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077630.449629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913500288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077630.449633:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913500416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077630.449649:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077630.449660:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077630/real 1769077630] req@ffff935002a61500 x1855008913500416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077630.449673:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913500416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077630.449682:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077630.449686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077630.449690:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913500416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077630.449697:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913500544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077630.449716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077630.449725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077630/real 1769077630] req@ffff935002a62300 x1855008913500544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077630.449733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913500544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077630.449739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077630.449741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077630.449743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913500544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077630.449747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500672:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077630.449757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077630.449763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077630/real 1769077630] req@ffff935002a61880 x1855008913500672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077630.449771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913500672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077630.449776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077630.449778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077630.449780:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077630.449784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077630.449793:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077630.449799:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077630/real 1769077630] req@ffff935002a61180 x1855008913500800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077630.449814:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913500800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077630.449821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077630.449823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077630.449825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077630.449829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077630.449841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077630.449847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077630/real 1769077630] req@ffff935002a62680 x1855008913500928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077630.449855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913500928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077630.449860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077630.449862:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077630.449864:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913500928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077630.641188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077631.153196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077631.153197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077631.153263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077631.153273:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077631.473143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077631.665167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077632.177169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077632.177169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077632.177169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077632.177200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077632.497195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077632.689176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077633.201161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077633.201181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077633.201182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077633.201255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077633.521146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077633.713169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077634.225146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077634.225172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077634.225176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077634.225337:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077634.545199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077634.737201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077635.249179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077635.249179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077635.249212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077635.249345:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077635.569176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077635.569186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077635.569198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077635.569205:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077635.569218:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3341 00000100:00080000:2.0:1769077635.569231:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077635.569241:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077635.569290:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913501056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077635.569327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077635.569332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077635.569334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077635.569339:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3341 00000100:00080000:2.0:1769077635.569343:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077635.569347:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077635.569359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077635.569381:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913501184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077635.569383:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913501056:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077635.569415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077635.569418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077635.569420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077635.569442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3341 00000100:00080000:2.0:1769077635.569448:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077635.569453:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077635.569459:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077635.569472:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913501312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077635.569480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077635.569483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077635.569488:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077635.569489:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077635.569500:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3341 00000100:00080000:2.0:1769077635.569506:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077635.569510:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077635.569515:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077635.569528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913501440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077635.569536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077635.569539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077635.569541:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077635.569545:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077635/real 1769077635] req@ffff935002a60a80 x1855008913501056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077635.569560:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3341 00000100:00080000:2.0:1769077635.569567:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077635.569571:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077635.569577:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913501056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077635.569581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077635.569605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913501568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077635.569606:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077635.569626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077635.569633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077635.569641:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913501056:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077635.569643:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077635.569648:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077635.569666:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3341 00000100:00080000:2.0:1769077635.569671:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077635.569674:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077635.569677:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077635.569690:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913501184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077635.569690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913501696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077635.569720:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077635.569722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077635.569725:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077635.569732:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3341 00000100:00080000:2.0:1769077635.569737:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077635.569741:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077635.569743:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077635.569751:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077635.569763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077635/real 1769077635] req@ffff935002a63100 x1855008913501184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077635.569765:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913501824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077635.569793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913501184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077635.569802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077635.569806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077635.569814:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913501184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077635.569821:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913501312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077635.569835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077635.569842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077635/real 1769077635] req@ffff935002a61c00 x1855008913501312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077635.569850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913501312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077635.569856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077635.569858:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077635.569861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913501312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077635.569866:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913501440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077635.569878:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077635.569885:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077635/real 1769077635] req@ffff935002a61f80 x1855008913501440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077635.569894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913501440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077635.569901:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077635.569903:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077635.569905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913501440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077635.569909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913501568:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077635.569919:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077635.569925:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077635/real 1769077635] req@ffff935002a62680 x1855008913501568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077635.569931:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913501568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077635.569937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077635.569939:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077635.569941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913501568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077635.569954:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913501696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077635.569964:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077635.569969:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077635/real 1769077635] req@ffff935002a61180 x1855008913501696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077635.569977:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913501696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077635.569984:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077635.569986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077635.569988:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913501696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077635.569993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913501824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077635.570002:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077635.570009:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077635/real 1769077635] req@ffff935002a61880 x1855008913501824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077635.570017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913501824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077635.570022:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077635.570023:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077635.570025:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913501824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077635.761128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077636.273134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077636.273148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077636.273219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077636.273242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077636.593167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077636.785129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077637.297175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077637.297175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077637.297189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077637.297260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077637.617167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077637.809098:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077638.321113:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077638.321122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077638.321128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077638.321238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077638.641185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077638.833190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077639.345168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077639.345187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077639.345227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077639.345280:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077639.665178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077639.857154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077640.369186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077640.369204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077640.369239:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077640.369272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077640.689182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077640.689225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077640.689232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077640.689236:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077640.689244:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3346 00000100:00080000:0.0:1769077640.689255:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077640.689261:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077640.689295:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913501952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077640.689319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077640.689322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077640.689323:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077640.689326:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3346 00000100:00080000:0.0:1769077640.689328:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077640.689331:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077640.689352:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077640.689369:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913502080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077640.689378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077640.689380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077640.689382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077640.689386:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3346 00000100:00080000:0.0:1769077640.689390:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077640.689394:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077640.689400:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077640.689402:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913501952:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077640.689410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913502208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077640.689431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077640.689433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077640.689434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077640.689436:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3346 00000100:00080000:0.0:1769077640.689438:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077640.689440:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077640.689443:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077640.689452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913502336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077640.689457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077640.689460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077640.689461:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077640.689464:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3346 00000100:00080000:0.0:1769077640.689466:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077640.689469:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077640.689472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077640.689479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913502464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077640.689485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077640.689487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077640.689489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077640.689491:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3346 00000100:00080000:0.0:1769077640.689496:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077640.689501:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077640.689507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077640.689516:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077640.689524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913502592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077640.689536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077640.689540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077640.689542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077640.689547:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3346 00000100:00080000:0.0:1769077640.689551:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077640.689555:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077640.689562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077640.689578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913502720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077640.689582:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077640/real 1769077640] req@ffff9350103fb480 x1855008913501952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077640.689632:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913501952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077640.689648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077640.689654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077640.689658:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913501952:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077640.689668:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502080:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077640.689694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077640.689707:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077640/real 1769077640] req@ffff9350103faa00 x1855008913502080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077640.689732:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913502080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077640.689742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077640.689746:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077640.689750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077640.689784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077640.689806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077640.689819:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077640/real 1769077640] req@ffff9350103fb100 x1855008913502208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077640.689833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913502208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077640.689842:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077640.689846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077640.689850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077640.689857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077640.689875:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077640.689886:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077640/real 1769077640] req@ffff9350103f8700 x1855008913502336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077640.689900:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913502336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077640.689909:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077640.689913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077640.689916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077640.689923:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913502464:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077640.689942:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077640.689953:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077640/real 1769077640] req@ffff9350103fad80 x1855008913502464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077640.689967:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913502464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077640.689976:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077640.689980:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077640.689984:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913502464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077640.689990:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913502592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077640.690008:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077640.690018:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077640/real 1769077640] req@ffff9350103fb800 x1855008913502592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077640.690032:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913502592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077640.690041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077640.690045:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077640.690074:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913502592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077640.690083:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913502720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077640.690103:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077640.690115:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077640/real 1769077640] req@ffff9350103f8380 x1855008913502720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077640.690128:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913502720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077640.690137:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077640.690141:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077640.690146:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913502720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077640.881157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077641.393181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077641.393181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077641.393187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077641.393278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077641.713165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077641.905159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077642.417167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077642.417167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077642.417167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077642.417247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077642.737188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077642.929180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077643.441204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077643.441205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077643.441227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077643.441393:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077643.761147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077643.953148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077644.465142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077644.465142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077644.465176:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077644.465269:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077644.785144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077644.977126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077645.489161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077645.489161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077645.489188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077645.489196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077645.809128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077645.809128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077645.809139:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077645.809145:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077645.809155:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3351 00000100:00080000:0.0:1769077645.809167:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077645.809175:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077645.809214:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913502848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077645.809243:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077645.809247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077645.809249:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077645.809267:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3351 00000100:00080000:0.0:1769077645.809271:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077645.809276:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077645.809287:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077645.809302:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913502976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077645.809314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077645.809317:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913502848:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077645.809319:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077645.809323:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077645.809330:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3351 00000100:00080000:0.0:1769077645.809333:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077645.809337:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077645.809343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077645.809357:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913503104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077645.809366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077645.809369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077645.809371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077645.809375:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3351 00000100:00080000:0.0:1769077645.809379:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077645.809382:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077645.809387:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077645.809393:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077645.809401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913503232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077645.809413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077645.809417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077645.809419:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077645.809423:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3351 00000100:00080000:0.0:1769077645.809427:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077645.809432:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077645.809438:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077645.809438:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077645/real 1769077645] req@ffff9350103f9180 x1855008913502848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077645.809456:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913503360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077645.809465:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913502848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077645.809473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077645.809481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077645.809484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077645.809486:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077645.809490:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3351 00000100:00080000:0.0:1769077645.809494:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077645.809495:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077645.809500:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077645.809504:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913502848:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769077645.809507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077645.809521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913503488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077645.809529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077645.809532:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502976:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077645.809533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077645.809536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077645.809542:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3351 00000100:00080000:0.0:1769077645.809545:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077645.809547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077645.809551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077645.809562:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077645.809564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913503616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077645.809578:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077645/real 1769077645] req@ffff9350103f9f80 x1855008913502976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077645.809588:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913502976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077645.809594:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077645.809596:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077645.809599:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913502976:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077645.809604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913503104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077645.809616:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077645.809623:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077645/real 1769077645] req@ffff9350103f9c00 x1855008913503104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077645.809635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913503104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077645.809644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077645.809648:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077645.809652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913503104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077645.809658:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913503232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077645.809679:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077645.809691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077645/real 1769077645] req@ffff9350103fbb80 x1855008913503232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077645.809704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913503232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077645.809714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077645.809718:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077645.809721:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913503232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077645.809727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913503360:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077645.809743:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077645.809752:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077645/real 1769077645] req@ffff9350103f9500 x1855008913503360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077645.809764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913503360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077645.809772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077645.809775:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077645.809778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913503360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077645.809795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913503488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077645.809810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077645.809819:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077645/real 1769077645] req@ffff9350103f8380 x1855008913503488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077645.809832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913503488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077645.809840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077645.809842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077645.809845:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913503488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077645.809851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913503616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077645.809864:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077645.809871:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077645/real 1769077645] req@ffff9350103fb800 x1855008913503616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077645.809879:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913503616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077645.809885:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077645.809887:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077645.809889:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913503616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077646.001153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077646.513134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077646.513134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077646.513134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077646.513186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077646.833144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077647.025134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077647.537181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077647.537188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077647.537194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077647.537260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077647.857187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077648.049169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077648.561122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077648.561129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077648.561140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077648.561169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077648.881121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077649.073123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077649.585134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077649.585135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077649.585153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077649.585168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077649.905172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077650.097149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077650.609213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077650.609248:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077650.609250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077650.609294:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077650.929156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077650.929192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.929202:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077650.929206:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.929215:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3356 00000100:00080000:0.0:1769077650.929226:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.929232:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077650.929266:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913503744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077650.929292:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.929295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077650.929297:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.929300:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3356 00000100:00080000:0.0:1769077650.929303:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.929306:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077650.929314:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077650.929325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913503872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077650.929334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.929338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077650.929338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913503744:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077650.929342:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.929350:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3356 00000100:00080000:0.0:1769077650.929352:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.929355:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077650.929359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077650.929369:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913504000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077650.929376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.929378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077650.929380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.929383:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3356 00000100:00080000:0.0:1769077650.929385:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.929387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077650.929389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077650.929397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913504128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077650.929403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.929406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077650.929407:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.929411:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3356 00000100:00080000:0.0:1769077650.929413:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.929416:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077650.929419:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077650.929427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913504256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077650.929434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.929437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077650.929440:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077650.929443:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913503872:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077650.929446:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3356 00000100:00080000:0.0:1769077650.929453:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.929456:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077650.929460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077650.929473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913504384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077650.929480:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504000:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077650.929482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.929488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077650.929490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.929499:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3356 00000100:00080000:0.0:1769077650.929505:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.929508:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077650.929511:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504128:192.168.204.121@tcp:8:kworker.0 00000008:00100000:0.0:1769077650.929514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077650.929532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913504512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077650.929538:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913504256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077650.929562:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913504384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077650.929579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913504512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077651.121127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077651.633182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077651.633194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077651.633199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077651.633252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077651.953209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077652.145168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077652.657189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077652.657189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077652.657189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077652.657236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077652.977215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077653.169155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077653.681142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077653.681143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077653.681187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077653.681193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077654.001184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077654.001355:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077654.001401:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077654.001422:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077654.001478:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077654.001496:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077654.001506:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077654.001518:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077654.001526:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077654] req@ffff9350103f8700 x1855008913503872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769077654.001529:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077654.001543:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077654.001562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913503872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077654.001584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077654.001591:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077654.001596:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913503872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077654.001605:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077654] req@ffff9350103fb100 x1855008913504000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077654.001636:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913504000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077654.001646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077654.001650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077654.001654:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077654.001662:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077654] req@ffff9350103faa00 x1855008913504128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077654.001675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913504128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077654.001684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077654.001688:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077654.001692:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077654.001701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077654] req@ffff9350103fb480 x1855008913504256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077654.001716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913504256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077654.001724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077654.001728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077654.001732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913504256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077654.001739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077654] req@ffff9350103fad80 x1855008913503744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077654.001752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913503744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077654.001760:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077654.001764:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077654.001767:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913503744:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077654.001803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077654] req@ffff9350103fb800 x1855008913504384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077654.001815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913504384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077654.001823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077654.001827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077654.001830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913504384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077654.001837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077654] req@ffff9350103f8380 x1855008913504512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077654.001849:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913504512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077654.001857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077654.001860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077654.001864:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913504512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077654.193183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077654.705161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077654.705161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077654.705196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077654.705208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077655.025135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077655.217157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077655.729127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077655.729128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077655.729135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077655.729193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077656.049168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077656.049180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077656.049187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077656.049192:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077656.049203:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3361 00000100:00080000:2.0:1769077656.049214:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077656.049222:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077656.049259:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913504640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077656.049288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077656.049291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077656.049292:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077656.049296:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3361 00000100:00080000:2.0:1769077656.049299:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077656.049302:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077656.049311:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077656.049327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913504768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077656.049340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077656.049344:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913504640:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077656.049346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077656.049351:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077656.049358:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3361 00000100:00080000:2.0:1769077656.049363:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077656.049368:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077656.049374:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077656.049388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913504896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077656.049399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077656.049403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077656.049405:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077656.049409:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3361 00000100:00080000:2.0:1769077656.049423:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077656.049430:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077656.049431:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769077656.049440:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077656.049456:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913505024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077656.049468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077656.049473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077656.049475:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077656.049478:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077656/real 1769077656] req@ffff935002a62300 x1855008913504640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077656.049483:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3361 00000100:00080000:2.0:1769077656.049491:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077656.049498:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077656.049505:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077656.049506:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913504640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077656.049522:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769077656.049524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913505152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077656.049530:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077656.049539:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913504640:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077656.049543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077656.049548:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077656.049550:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077656.049554:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3361 00000100:00080000:2.0:1769077656.049559:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077656.049562:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077656.049567:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077656.049572:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077656.049578:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913505280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077656.049589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077656.049594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077656.049596:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077656.049600:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3361 00000800:00000100:0.0:1769077656.049607:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077656.049607:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077656.049615:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077656.049621:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077656.049628:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077656/real 1769077656] req@ffff935002a61500 x1855008913504768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077656.049637:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913505408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077656.049681:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913504768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077656.049691:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077656.049695:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077656.049699:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077656.049706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077656.049731:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077656.049745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077656/real 1769077656] req@ffff935002a62a00 x1855008913504896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077656.049758:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913504896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077656.049769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077656.049773:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077656.049776:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913504896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077656.049781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077656.049801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077656.049812:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077656/real 1769077656] req@ffff935002a60000 x1855008913505024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077656.049824:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913505024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077656.049839:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077656.049842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077656.049846:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077656.049851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913505152:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077656.049870:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077656.049882:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077656/real 1769077656] req@ffff935002a61880 x1855008913505152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077656.049894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913505152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077656.049902:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077656.049906:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077656.049909:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913505152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077656.049927:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913505280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077656.049944:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077656.049956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077656/real 1769077656] req@ffff935002a61180 x1855008913505280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077656.049968:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913505280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077656.049977:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077656.049980:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077656.049984:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913505280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077656.049990:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913505408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077656.050009:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077656.050020:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077656/real 1769077656] req@ffff935002a62680 x1855008913505408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077656.050031:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913505408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077656.050039:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077656.050042:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077656.050046:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913505408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077656.241157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077656.753155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077656.753155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077656.753155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077656.753180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077657.073160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077657.265121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077657.777147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077657.777147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077657.777147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077657.777195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077658.097156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077658.289150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077658.801161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077658.801161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077658.801168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077658.801202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077659.121145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077659.313159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077659.825133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077659.825133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077659.825134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077659.825196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077660.145183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077660.337142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077660.849155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077660.849155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077660.849164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077660.849211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077661.169149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077661.169158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077661.169163:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077661.169164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077661.169174:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3366 00000100:00080000:3.0:1769077661.169187:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077661.169193:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077661.169227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913505536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077661.169251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077661.169254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077661.169255:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077661.169259:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3366 00000100:00080000:3.0:1769077661.169261:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077661.169263:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077661.169270:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077661.169279:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913505664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077661.169287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077661.169291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077661.169293:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077661.169299:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913505536:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077661.169301:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3366 00000100:00080000:3.0:1769077661.169308:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077661.169314:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077661.169320:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077661.169334:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913505792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077661.169346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077661.169350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077661.169352:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077661.169356:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3366 00000100:00080000:3.0:1769077661.169360:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077661.169363:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077661.169368:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077661.169380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913505920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077661.169381:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077661.169390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077661.169394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077661.169396:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077661.169399:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3366 00000100:00080000:3.0:1769077661.169404:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077661.169406:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077661.169409:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077661.169420:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913506048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077661.169426:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077661/real 1769077661] req@ffff935002a7c700 x1855008913505536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077661.169450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077661.169453:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077661.169454:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077661.169458:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3366 00000100:00080000:3.0:1769077661.169461:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077661.169462:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913505536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077661.169466:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077661.169473:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077661.169483:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077661.169484:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913506176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077661.169495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077661.169496:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077661.169498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077661.169501:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077661.169506:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3366 00000100:00080000:3.0:1769077661.169508:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077661.169509:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913505536:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077661.169511:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077661.169516:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077661.169525:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913506304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077661.169542:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505664:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077661.169565:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077661.169575:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077661/real 1769077661] req@ffff935002a7e300 x1855008913505664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077661.169584:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913505664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077661.169590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077661.169593:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077661.169595:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077661.169599:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077661.169610:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077661.169616:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077661/real 1769077661] req@ffff935002a7ca80 x1855008913505792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077661.169623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913505792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077661.169628:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077661.169630:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077661.169632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077661.169636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077661.169645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077661.169651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077661/real 1769077661] req@ffff935002a7c380 x1855008913505920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077661.169659:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913505920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077661.169665:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077661.169666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077661.169668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913505920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077661.169671:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506048:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077661.169686:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077661.169696:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077661/real 1769077661] req@ffff935002a7f100 x1855008913506048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077661.169709:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913506048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077661.169718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077661.169722:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077661.169725:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077661.169733:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077661.169751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077661.169760:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077661/real 1769077661] req@ffff935002a7fb80 x1855008913506176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077661.169768:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913506176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077661.169774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077661.169787:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077661.169789:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077661.169794:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077661.169806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077661.169813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077661/real 1769077661] req@ffff935002a7df80 x1855008913506304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077661.169820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913506304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077661.169826:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077661.169828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077661.169830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077661.361150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077661.873131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077661.873131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077661.873131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077661.873167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077662.193191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077662.385121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077662.897131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077662.897136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077662.897142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077662.897182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077663.217124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077663.409162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077663.921137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077663.921137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077663.921140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077663.921174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077664.241121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077664.433120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077664.945203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077664.945203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077664.945272:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077664.945273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077665.265162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077665.457173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077665.969135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077665.969154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077665.969207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077665.969212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077666.289142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077666.289145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077666.289154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077666.289158:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077666.289166:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3371 00000100:00080000:3.0:1769077666.289176:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077666.289182:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077666.289227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913506432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077666.289249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077666.289252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077666.289254:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077666.289257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3371 00000100:00080000:3.0:1769077666.289264:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077666.289266:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077666.289273:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077666.289282:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913506560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077666.289289:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077666.289292:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077666.289295:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077666.289298:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3371 00000100:00080000:3.0:1769077666.289301:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077666.289303:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077666.289307:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077666.289316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913506688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077666.289322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077666.289325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077666.289327:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077666.289331:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3371 00000100:00080000:3.0:1769077666.289332:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077666.289334:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077666.289337:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077666.289345:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913506816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077666.289352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077666.289354:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077666.289356:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077666.289363:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3371 00000100:00080000:3.0:1769077666.289366:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077666.289368:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913506432:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077666.289370:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077666.289378:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077666.289398:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913506944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077666.289407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077666.289410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077666.289412:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077666.289414:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3371 00000100:00080000:3.0:1769077666.289416:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077666.289418:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077666.289422:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077666.289430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913507072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077666.289437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077666.289439:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077666.289440:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077666.289443:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3371 00000100:00080000:3.0:1769077666.289448:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077666.289453:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077666.289458:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077666.289473:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913507200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077666.289475:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077666.289556:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077666/real 1769077666] req@ffff935002a7c000 x1855008913506432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077666.289583:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913506432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077666.289597:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077666.289605:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077666.289610:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913506432:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077666.289622:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913506560:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077666.289655:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077666.289672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077666/real 1769077666] req@ffff935002a7f800 x1855008913506560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077666.289688:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913506560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077666.289697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077666.289701:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077666.289704:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913506560:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077666.289710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913506688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077666.289726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077666.289735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077666/real 1769077666] req@ffff935002a7dc00 x1855008913506688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077666.289746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913506688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077666.289754:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077666.289758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077666.289761:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913506688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077666.289825:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913506816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077666.289854:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077666.289871:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077666/real 1769077666] req@ffff935002a7d180 x1855008913506816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077666.289886:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913506816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077666.289896:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077666.289901:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077666.289905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913506816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077666.289930:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506944:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077666.289959:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077666.289974:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077666/real 1769077666] req@ffff935002a7df80 x1855008913506944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077666.289989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913506944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077666.289999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077666.290004:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077666.290008:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913506944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077666.290017:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077666.290032:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077666.290041:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077666/real 1769077666] req@ffff935002a7fb80 x1855008913507072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077666.290096:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913507072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077666.290105:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077666.290108:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077666.290111:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077666.290117:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077666.290136:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077666.290147:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077666/real 1769077666] req@ffff935002a7f100 x1855008913507200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077666.290159:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913507200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077666.290169:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077666.290172:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077666.290176:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077666.481111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077666.993138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077666.993138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077666.993141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077666.993181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077667.313161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077667.505161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077668.017121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077668.017129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077668.017138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077668.017173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077668.337137:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077668.529162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077669.041145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077669.041145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077669.041158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077669.041183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077669.361182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077669.553161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077670.065158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077670.065158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077670.065158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077670.065183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077670.385136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077670.577122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077671.089124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077671.089132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077671.089149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077671.089190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077671.409172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077671.409241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077671.409249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077671.409253:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077671.409264:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3377 00000100:00080000:0.0:1769077671.409274:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077671.409280:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077671.409317:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913507328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077671.409349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077671.409353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077671.409356:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077671.409360:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3377 00000100:00080000:0.0:1769077671.409364:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077671.409367:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077671.409376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077671.409388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913507456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077671.409396:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913507328:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077671.409400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077671.409404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077671.409406:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077671.409409:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3377 00000100:00080000:0.0:1769077671.409411:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077671.409414:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077671.409417:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077671.409427:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913507584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077671.409438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077671.409441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077671.409443:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077671.409445:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3377 00000100:00080000:0.0:1769077671.409447:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077671.409449:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077671.409452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077671.409459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913507712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077671.409465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077671.409467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077671.409469:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077671.409475:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3377 00000800:00000100:2.0:1769077671.409478:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077671.409480:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077671.409485:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077671.409488:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077671.409497:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913507840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077671.409506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077671.409510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077671.409512:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077671.409516:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3377 00000100:00080000:0.0:1769077671.409520:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077671.409524:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077671.409529:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077671.409544:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077671/real 1769077671] req@ffff9350103f9500 x1855008913507328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077671.409546:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913507968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077671.409580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077671.409582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077671.409584:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077671.409589:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913507328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077671.409590:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3377 00000100:00080000:0.0:1769077671.409597:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077671.409604:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077671.409609:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769077671.409610:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077671.409621:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077671.409627:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913507328:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077671.409640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913508096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077671.409655:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913507456:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077671.409685:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077671.409696:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077671/real 1769077671] req@ffff9350103fbb80 x1855008913507456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077671.409723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913507456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077671.409732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077671.409736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077671.409740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913507456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077671.409748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913507584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077671.409778:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077671.409789:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077671/real 1769077671] req@ffff9350103f9c00 x1855008913507584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077671.409798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913507584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077671.409804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077671.409806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077671.409809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913507584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077671.409813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913507712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077671.409857:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077671.409866:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077671/real 1769077671] req@ffff9350103f9f80 x1855008913507712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077671.409875:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913507712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077671.409881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077671.409884:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077671.409886:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913507712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077671.409890:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507840:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077671.409903:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077671.409909:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077671/real 1769077671] req@ffff9350103f9180 x1855008913507840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077671.409925:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913507840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077671.409932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077671.409934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077671.409936:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507840:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077671.409940:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077671.409952:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077671.409958:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077671/real 1769077671] req@ffff9350103f8380 x1855008913507968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077671.409966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913507968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077671.409972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077671.409976:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077671.409980:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913507968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077671.410000:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077671.410021:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077671.410032:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077671/real 1769077671] req@ffff9350103fb800 x1855008913508096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077671.410042:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913508096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077671.410074:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077671.410079:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077671.410083:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077671.601164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077672.113146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077672.113146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077672.113174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077672.113183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077672.433136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077672.625120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077673.137178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077673.137178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077673.137181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077673.137229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077673.457166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077673.649132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077674.161147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077674.161147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077674.161150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077674.161177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077674.481145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077674.673121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077675.185108:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077675.185130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077675.185138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077675.185165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077675.505168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077675.697119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077676.209115:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077676.209115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077676.209127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077676.209174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077676.529130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077676.529145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077676.529151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077676.529155:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077676.529163:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3382 00000100:00080000:3.0:1769077676.529174:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077676.529180:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077676.529212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913508224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077676.529235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077676.529238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077676.529244:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077676.529247:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3382 00000100:00080000:3.0:1769077676.529250:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077676.529253:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077676.529261:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077676.529271:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913508352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077676.529277:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913508224:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077676.529281:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077676.529285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077676.529286:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077676.529289:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3382 00000100:00080000:3.0:1769077676.529291:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077676.529293:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077676.529297:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077676.529306:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913508480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077676.529313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077676.529315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077676.529317:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077676.529319:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3382 00000100:00080000:3.0:1769077676.529333:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077676.529335:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077676.529338:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077676.529343:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077676.529348:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913508608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077676.529356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077676.529359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077676.529360:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077676.529364:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3382 00000100:00080000:3.0:1769077676.529366:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077676.529368:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077676.529372:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077676.529380:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077676/real 1769077676] req@ffff935002a7c380 x1855008913508224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077676.529383:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913508736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077676.529408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077676.529411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077676.529414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077676.529421:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3382 00000100:00100000:2.0:1769077676.529422:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913508224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077676.529424:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077676.529431:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077676.529441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769077676.529446:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077676.529447:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077676.529453:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913508224:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077676.529460:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913508864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077676.529467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077676.529470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077676.529472:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077676.529476:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913508352:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077676.529478:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3382 00000100:00080000:3.0:1769077676.529484:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077676.529487:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077676.529491:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077676.529503:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077676.529504:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913508992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077676.529517:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077676/real 1769077676] req@ffff935002a7ca80 x1855008913508352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077676.529530:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913508352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077676.529542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077676.529544:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077676.529546:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913508352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077676.529551:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913508480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077676.529562:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077676.529568:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077676/real 1769077676] req@ffff935002a7e300 x1855008913508480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077676.529575:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913508480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077676.529581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077676.529582:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077676.529584:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913508480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077676.529588:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913508608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077676.529597:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077676.529602:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077676/real 1769077676] req@ffff935002a7c700 x1855008913508608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077676.529609:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913508608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077676.529614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077676.529616:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077676.529619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913508608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077676.529622:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508736:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077676.529632:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077676.529636:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077676/real 1769077676] req@ffff935002a7f100 x1855008913508736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077676.529644:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913508736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077676.529650:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077676.529651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077676.529653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077676.529657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077676.529666:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077676.529671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077676/real 1769077676] req@ffff935002a7fb80 x1855008913508864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077676.529679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913508864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077676.529684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077676.529685:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077676.529687:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077676.529700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077676.529709:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077676.529715:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077676/real 1769077676] req@ffff935002a7df80 x1855008913508992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077676.529723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913508992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077676.529728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077676.529730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077676.529732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913508992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077676.721142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077677.233135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077677.233135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077677.233148:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077677.233174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077677.553162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077677.745197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077678.257158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077678.257159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077678.257159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077678.257193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077678.577146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077678.769117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077679.281130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077679.281134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077679.281138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077679.281157:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077679.601151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077679.793131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077680.305159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077680.305160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077680.305163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077680.305193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077680.625173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077680.817150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077681.329165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077681.329165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077681.329165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077681.329191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077681.649160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077681.649166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077681.649171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077681.649177:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077681.649185:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3387 00000100:00080000:3.0:1769077681.649195:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077681.649201:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077681.649251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913509120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077681.649279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077681.649284:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077681.649286:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077681.649291:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3387 00000100:00080000:3.0:1769077681.649295:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077681.649299:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077681.649309:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077681.649323:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913509248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077681.649334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077681.649338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077681.649340:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077681.649344:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3387 00000100:00080000:3.0:1769077681.649348:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077681.649352:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077681.649358:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913509120:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077681.649360:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077681.649377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913509376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077681.649389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077681.649393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077681.649395:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077681.649399:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3387 00000100:00080000:3.0:1769077681.649415:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077681.649419:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077681.649424:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077681.649425:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077681.649443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913509504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077681.649454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077681.649457:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077681.649459:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077681.649462:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077681/real 1769077681] req@ffff935002a7d180 x1855008913509120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077681.649466:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3387 00000100:00080000:3.0:1769077681.649472:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077681.649478:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077681.649484:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077681.649486:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913509120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077681.649496:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077681.649499:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913509632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077681.649502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077681.649510:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913509120:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077681.649514:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077681.649518:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077681.649520:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913509248:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077681.649521:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077681.649529:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3387 00000100:00080000:3.0:1769077681.649532:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077681.649535:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077681.649538:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077681.649547:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077681.649550:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913509760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077681.649559:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077681.649562:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077681/real 1769077681] req@ffff935002a7dc00 x1855008913509248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077681.649563:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077681.649567:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077681.649575:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3387 00000100:00080000:3.0:1769077681.649581:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077681.649585:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913509248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077681.649588:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077681.649594:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077681.649600:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077681.649606:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913509888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077681.649607:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077681.649623:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913509248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077681.649644:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913509376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077681.649693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077681.649702:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077681/real 1769077681] req@ffff935002a7f800 x1855008913509376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077681.649711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913509376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077681.649717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077681.649719:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077681.649722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913509376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077681.649726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913509504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077681.649736:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077681.649741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077681/real 1769077681] req@ffff935002a7c000 x1855008913509504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077681.649749:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913509504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077681.649755:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077681.649757:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077681.649759:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913509504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077681.649762:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913509632:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077681.649771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077681.649776:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077681/real 1769077681] req@ffff935002a7df80 x1855008913509632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077681.649790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913509632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077681.649796:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077681.649798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077681.649801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913509632:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077681.649804:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913509760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077681.649815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077681.649821:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077681/real 1769077681] req@ffff935002a7fb80 x1855008913509760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077681.649829:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913509760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077681.649834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077681.649836:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077681.649838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913509760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077681.649842:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913509888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077681.649852:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077681.649857:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077681/real 1769077681] req@ffff935002a7f100 x1855008913509888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077681.649864:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913509888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077681.649869:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077681.649871:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077681.649873:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913509888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077681.841116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077682.353150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077682.353150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077682.353150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077682.353185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077682.673152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077682.865125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077683.377131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077683.377135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077683.377138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077683.377158:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077683.697107:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077683.889108:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077684.401155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077684.401155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077684.401169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077684.401180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077684.721150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077684.913178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077685.425131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077685.425143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077685.425143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077685.425186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077685.745169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077685.937108:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077686.449151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077686.449152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077686.449172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077686.449186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077686.769165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077686.769189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077686.769202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077686.769208:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077686.769220:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3392 00000100:00080000:3.0:1769077686.769234:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077686.769243:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077686.769291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913510016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077686.769332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077686.769337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077686.769340:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077686.769345:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3392 00000100:00080000:3.0:1769077686.769349:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077686.769354:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077686.769366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077686.769382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913510144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077686.769391:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913510016:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077686.769400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077686.769405:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077686.769408:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077686.769413:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3392 00000100:00080000:3.0:1769077686.769418:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077686.769423:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077686.769439:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077686.769454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913510272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077686.769465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077686.769466:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077686.769473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077686.769476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077686.769483:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3392 00000100:00080000:3.0:1769077686.769491:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077686.769496:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077686.769502:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077686.769513:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077686/real 1769077686] req@ffff935002a7c700 x1855008913510016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077686.769519:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913510400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077686.769536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077686.769539:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913510016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077686.769542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077686.769548:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077686.769554:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077686.769556:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3392 00000100:00080000:2.0:1769077686.769560:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077686.769577:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077686.769586:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077686.769590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913510016:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077686.769593:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077686.769611:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913510528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077686.769618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913510144:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077686.769632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077686.769636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077686.769639:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077686.769645:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3392 00000100:00080000:3.0:1769077686.769651:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077686.769654:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077686.769660:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077686.769668:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077686.769670:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077686/real 1769077686] req@ffff935002a7e300 x1855008913510144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077686.769686:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913510144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077686.769688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913510656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077686.769702:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077686.769709:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077686.769711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077686.769713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913510144:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077686.769722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077686.769724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913510272:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077686.769726:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077686.769735:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3392 00000100:00080000:3.0:1769077686.769740:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077686.769742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077686.769749:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077686.769757:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077686.769762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077686/real 1769077686] req@ffff935002a7ca80 x1855008913510272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077686.769773:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913510784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077686.769774:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913510272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077686.769803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077686.769807:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077686.769810:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913510272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077686.769814:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913510400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077686.769827:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077686.769835:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077686/real 1769077686] req@ffff935002a7c380 x1855008913510400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077686.769843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913510400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077686.769848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077686.769850:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077686.769852:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913510400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077686.769862:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913510528:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077686.769871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077686.769877:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077686/real 1769077686] req@ffff935002a7f100 x1855008913510528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077686.769885:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913510528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077686.769890:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077686.769891:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077686.769893:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913510528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077686.769897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913510656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077686.769905:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077686.769911:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077686/real 1769077686] req@ffff935002a7fb80 x1855008913510656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077686.769918:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913510656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077686.769927:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077686.769929:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077686.769931:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913510656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077686.769934:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913510784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077686.769942:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077686.769947:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077686/real 1769077686] req@ffff935002a7df80 x1855008913510784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077686.769954:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913510784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077686.769958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077686.769960:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077686.769962:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913510784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077686.961150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077687.473135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077687.473137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077687.473170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077687.473181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077687.793138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077687.985177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077688.497132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077688.497135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077688.497144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077688.497227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077688.817145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077689.009124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077689.521171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077689.521171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077689.521172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077689.521229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077689.841123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077690.033177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077690.545168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077690.545168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077690.545169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077690.545229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077690.865153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077691.057165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077691.569194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077691.569194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077691.569194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077691.569231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077691.889148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077691.889158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077691.889164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077691.889169:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077691.889179:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3397 00000100:00080000:3.0:1769077691.889191:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077691.889199:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077691.889256:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913510912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077691.889287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077691.889291:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077691.889294:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077691.889299:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3397 00000100:00080000:3.0:1769077691.889303:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077691.889308:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077691.889319:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077691.889332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913511040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077691.889357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077691.889364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077691.889366:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913510912:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077691.889366:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077691.889375:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3397 00000100:00080000:3.0:1769077691.889382:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077691.889386:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077691.889390:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077691.889401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913511168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077691.889408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077691.889411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077691.889412:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077691.889416:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3397 00000100:00080000:3.0:1769077691.889418:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077691.889421:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077691.889425:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077691.889433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913511296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077691.889440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077691.889444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077691.889445:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077691.889448:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3397 00000100:00080000:3.0:1769077691.889450:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077691.889457:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077691.889457:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077691.889466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077691.889477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913511424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077691.889500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077691.889503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077691.889507:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077691.889512:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3397 00000100:00080000:3.0:1769077691.889516:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077691.889522:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077691.889528:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077691.889541:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff935002a7c000 x1855008913510912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077691.889548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913511552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077691.889570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077691.889575:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913510912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077691.889578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077691.889586:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077691.889591:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077691.889599:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3397 00000100:00080000:2.0:1769077691.889603:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077691.889610:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913510912:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077691.889610:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077691.889637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077691.889645:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511040:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077691.889659:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077691.889668:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077691.889677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913511680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077691.889684:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff935002a7f800 x1855008913511040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.889712:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913511040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.889720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.889722:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077691.889729:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077691.889738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077691.889761:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077691.889773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff935002a7dc00 x1855008913511168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.889797:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913511168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.889803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.889805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077691.889807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077691.889812:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077691.889828:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077691.889835:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff935002a7d180 x1855008913511296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.889844:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913511296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.889849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.889852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077691.889854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077691.889858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913511424:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077691.889867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077691.889873:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff935002a7df80 x1855008913511424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.889880:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913511424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.889886:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.889888:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077691.889890:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913511424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077691.889893:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913511552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077691.889902:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077691.889908:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff935002a7fb80 x1855008913511552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.889920:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913511552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.889926:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.889927:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077691.889929:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913511552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077691.889948:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913511680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077691.889958:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077691.889964:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff935002a7f100 x1855008913511680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.889972:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913511680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.889977:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.889978:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077691.889980:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913511680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077692.081162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077692.593150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077692.593153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077692.593167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077692.593276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077692.913217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077693.105180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077693.617235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077693.617235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077693.617236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077693.617288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077693.937171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077694.129162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077694.641124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077694.641143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077694.641162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077694.641170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077694.961199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077695.153222:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077695.665157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077695.665168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077695.665180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077695.665204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077695.985151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077696.177135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077696.689186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077696.689186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077696.689217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077696.689240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:0.0:1769077697.009177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077697.009177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077697.009188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077697.009193:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077697.009204:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3402 00000100:00080000:0.0:1769077697.009216:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077697.009223:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077697.009266:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913511808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077697.009294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077697.009297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077697.009299:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077697.009302:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3402 00000100:00080000:0.0:1769077697.009304:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077697.009308:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077697.009317:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077697.009328:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913511936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077697.009336:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077697.009339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077697.009340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077697.009343:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3402 00000100:00080000:0.0:1769077697.009345:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077697.009347:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077697.009351:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077697.009354:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913511808:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077697.009361:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913512064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077697.009369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077697.009372:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077697.009373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077697.009375:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3402 00000100:00080000:0.0:1769077697.009378:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077697.009380:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077697.009383:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077697.009390:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913512192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077697.009421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077697.009424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077697.009425:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077697.009429:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3402 00000100:00080000:0.0:1769077697.009432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077697.009435:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077697.009439:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077697.009444:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077697.009451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913512320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077697.009459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077697.009462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077697.009464:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077697.009467:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3402 00000100:00080000:0.0:1769077697.009469:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077697.009471:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077697.009474:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077697.009484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913512448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077697.009490:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077697/real 1769077697] req@ffff9350103fad80 x1855008913511808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077697.009500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077697.009510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077697.009515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077697.009521:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3402 00000100:00080000:0.0:1769077697.009526:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077697.009527:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913511808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077697.009535:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077697.009540:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769077697.009544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077697.009545:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077697.009552:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913511808:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077697.009559:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077697.009562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913512576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077697.009582:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077697.009594:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077697/real 1769077697] req@ffff9350103fb480 x1855008913511936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077697.009603:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913511936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077697.009610:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077697.009612:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077697.009614:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913511936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077697.009639:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077697.009659:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077697.009671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077697/real 1769077697] req@ffff9350103faa00 x1855008913512064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077697.009684:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913512064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077697.009700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077697.009703:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077697.009705:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077697.009710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077697.009724:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077697.009732:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077697/real 1769077697] req@ffff9350103fb100 x1855008913512192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077697.009740:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913512192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077697.009746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077697.009748:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077697.009750:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077697.009753:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913512320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077697.009762:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077697.009767:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077697/real 1769077697] req@ffff9350103f8700 x1855008913512320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077697.009774:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913512320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077697.009783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077697.009786:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077697.009791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913512320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077697.009797:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913512448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077697.009816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077697.009827:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077697/real 1769077697] req@ffff9350103fb800 x1855008913512448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077697.009836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913512448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077697.009841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077697.009843:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077697.009846:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913512448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077697.009850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913512576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077697.009863:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077697.009869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077697/real 1769077697] req@ffff9350103f8380 x1855008913512576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077697.009877:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913512576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077697.009882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077697.009884:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077697.009886:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913512576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077697.201149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077697.713191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077697.713195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077697.713213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077697.713249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077698.033184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077698.225136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077698.737155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077698.737155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077698.737163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077698.737219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077699.057145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077699.249143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077699.761116:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077699.761135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077699.761151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077699.761222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077700.081152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077700.273164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077700.785151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077700.785151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077700.785152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077700.785242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077701.105129:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077701.297112:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077701.809179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077701.809179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077701.809179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077701.809237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077702.129193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077702.129196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077702.129205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077702.129210:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077702.129219:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3407 00000100:00080000:3.0:1769077702.129231:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077702.129238:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077702.129272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913512704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077702.129295:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077702.129299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077702.129302:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077702.129325:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3407 00000100:00080000:3.0:1769077702.129337:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077702.129345:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077702.129357:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077702.129367:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913512704:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077702.129384:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913512832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077702.129396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077702.129398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077702.129400:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077702.129403:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3407 00000100:00080000:3.0:1769077702.129408:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077702.129414:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077702.129421:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077702.129435:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913512960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077702.129448:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077702.129463:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077702.129467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077702.129470:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077702.129478:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3407 00000100:00080000:3.0:1769077702.129484:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077702.129490:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077702.129497:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077702.129511:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077702/real 1769077702] req@ffff935002a7c380 x1855008913512704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077702.129514:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913513088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077702.129575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077702.129579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077702.129583:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077702.129585:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913512704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077702.129590:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3407 00000100:00080000:3.0:1769077702.129599:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077702.129611:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077702.129621:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077702.129626:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077702.129636:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913513216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077702.129638:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077702.129658:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913512704:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077702.129665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077702.129669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077702.129671:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077702.129675:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3407 00000100:00080000:3.0:1769077702.129679:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077702.129683:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077702.129689:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077702.129697:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077702.129706:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913513344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077702.129722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077702.129727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077702.129729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077702.129736:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3407 00000800:00000100:2.0:1769077702.129738:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077702.129746:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077702.129751:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077702.129757:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077702.129769:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077702/real 1769077702] req@ffff935002a7ca80 x1855008913512832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077702.129779:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913513472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077702.129808:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913512832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077702.129821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077702.129826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077702.129830:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077702.129838:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077702.129865:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077702.129880:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077702/real 1769077702] req@ffff935002a7e300 x1855008913512960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077702.129894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913512960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077702.129904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077702.129908:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077702.129912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913512960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077702.129920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077702.129942:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077702.129956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077702/real 1769077702] req@ffff935002a7c700 x1855008913513088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077702.129969:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913513088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077702.129978:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077702.129982:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077702.129986:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077702.129993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913513216:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077702.130016:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077702.130028:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077702/real 1769077702] req@ffff935002a7f100 x1855008913513216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077702.130041:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913513216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077702.130103:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077702.130108:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077702.130112:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913513216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077702.130139:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913513344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077702.130162:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077702.130176:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077702/real 1769077702] req@ffff935002a7fb80 x1855008913513344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077702.130191:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913513344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077702.130200:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077702.130204:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077702.130208:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913513344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077702.130215:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913513472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077702.130237:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077702.130250:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077702/real 1769077702] req@ffff935002a7df80 x1855008913513472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077702.130264:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913513472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077702.130272:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077702.130276:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077702.130280:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913513472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077702.321166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077702.833133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077702.833133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077702.833144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077702.833224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077703.153145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077703.345116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077703.857178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077703.857179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077703.857186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077703.857219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077704.177220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077704.369156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077704.881165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077704.881165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077704.881166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077704.881274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077705.201167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077705.393158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077705.905153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077705.905153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077705.905267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077705.906363:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077706.225138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077706.417159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077706.929123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077706.929126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077706.929141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077706.929165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077707.249143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077707.249169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077707.249174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077707.249177:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077707.249184:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3412 00000100:00080000:1.0:1769077707.249193:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077707.249198:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077707.249252:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913513600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077707.249273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077707.249275:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077707.249277:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077707.249280:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3412 00000100:00080000:1.0:1769077707.249282:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077707.249285:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077707.249293:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077707.249304:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913513728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077707.249313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077707.249315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077707.249316:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077707.249316:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913513600:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077707.249322:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3412 00000100:00080000:1.0:1769077707.249325:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077707.249327:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077707.249331:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077707.249339:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913513856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077707.249344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077707.249346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077707.249348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077707.249350:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3412 00000100:00080000:1.0:1769077707.249353:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077707.249355:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077707.249357:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077707.249363:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913513984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077707.249368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077707.249370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077707.249372:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077707.249374:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3412 00000100:00080000:1.0:1769077707.249376:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077707.249378:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077707.249381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077707.249386:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077707.249391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913514112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077707.249397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077707.249400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077707.249402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077707.249404:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3412 00000100:00080000:1.0:1769077707.249407:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077707.249409:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077707.249412:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077707.249419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913514240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077707.249425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077707.249426:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077707/real 1769077707] req@ffff9350100cb800 x1855008913513600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077707.249429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077707.249433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077707.249439:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3412 00000100:00080000:1.0:1769077707.249443:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077707.249446:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077707.249449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077707.249452:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913513600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077707.249459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913514368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077707.249469:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077707.249474:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077707.249477:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913513600:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077707.249496:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077707.249513:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077707.249530:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077707/real 1769077707] req@ffff9350100ca300 x1855008913513728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077707.249542:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913513728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077707.249551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077707.249555:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077707.249558:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077707.249564:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077707.249581:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077707.249589:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077707/real 1769077707] req@ffff9350100c8380 x1855008913513856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077707.249603:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913513856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077707.249611:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077707.249621:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077707.249628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077707.249634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077707.249653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077707.249663:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077707/real 1769077707] req@ffff9350100caa00 x1855008913513984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077707.249674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913513984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077707.249683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077707.249686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077707.249689:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913513984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077707.249695:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913514112:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077707.249713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077707.249723:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077707/real 1769077707] req@ffff9350100c9500 x1855008913514112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077707.249731:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913514112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077707.249736:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077707.249738:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077707.249740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913514112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077707.249743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913514240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077707.249755:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077707.249764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077707/real 1769077707] req@ffff9350100c9880 x1855008913514240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077707.249774:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913514240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077707.249782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077707.249785:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077707.249788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913514240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077707.249794:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913514368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077707.249812:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077707.249820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077707/real 1769077707] req@ffff9350100c8e00 x1855008913514368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077707.249828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913514368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077707.249836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077707.249839:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077707.249840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913514368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077707.441145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077707.953153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077707.953153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077707.953270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077707.953340:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077708.273145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077708.465167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077708.977152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077708.977152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077708.977160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077708.977186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077709.297130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077709.489151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077710.001135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077710.001137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077710.001142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077710.001188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077710.321157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077710.513158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077711.025142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077711.025148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077711.025247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077711.025281:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077711.345104:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077711.537131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077712.049113:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077712.049115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077712.049149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077712.049185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077712.369182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077712.369197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077712.369205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077712.369209:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077712.369217:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3418 00000100:00080000:2.0:1769077712.369227:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077712.369232:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077712.369263:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913514496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077712.369293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077712.369295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077712.369296:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077712.369300:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3418 00000100:00080000:2.0:1769077712.369302:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077712.369304:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077712.369311:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077712.369320:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913514624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077712.369327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077712.369329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077712.369330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077712.369333:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3418 00000100:00080000:2.0:1769077712.369334:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077712.369336:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077712.369340:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077712.369342:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913514496:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077712.369348:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913514752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077712.369355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077712.369357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077712.369357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077712.369360:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3418 00000100:00080000:2.0:1769077712.369362:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077712.369364:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077712.369366:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077712.369373:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913514880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077712.369378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077712.369380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077712.369382:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077712.369384:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3418 00000100:00080000:2.0:1769077712.369386:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077712.369388:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077712.369391:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077712.369399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913515008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077712.369404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077712.369406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077712.369407:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077712.369409:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3418 00000100:00080000:2.0:1769077712.369412:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077712.369414:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077712.369417:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077712.369420:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077712.369428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913515136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077712.369433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077712.369435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077712.369436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077712.369439:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3418 00000100:00080000:2.0:1769077712.369441:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077712.369443:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077712.369446:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077712.369452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913515264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077712.369460:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077712/real 1769077712] req@ffff935002a61f80 x1855008913514496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077712.369474:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913514496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077712.369481:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077712.369486:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077712.369489:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913514496:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077712.369494:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913514624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077712.369509:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077712.369517:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077712/real 1769077712] req@ffff935002a61c00 x1855008913514624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077712.369524:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913514624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077712.369530:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077712.369532:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077712.369534:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913514624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077712.369551:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913514752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077712.369560:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077712.369565:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077712/real 1769077712] req@ffff935002a63100 x1855008913514752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077712.369572:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913514752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077712.369577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077712.369578:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077712.369580:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913514752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077712.369583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913514880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077712.369591:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077712.369596:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077712/real 1769077712] req@ffff935002a60a80 x1855008913514880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077712.369603:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913514880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077712.369607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077712.369609:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077712.369612:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913514880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077712.369616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515008:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077712.369633:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077712.369643:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077712/real 1769077712] req@ffff935002a62680 x1855008913515008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077712.369654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913515008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077712.369662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077712.369665:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077712.369668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077712.369673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077712.369687:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077712.369693:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077712/real 1769077712] req@ffff935002a61180 x1855008913515136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077712.369705:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913515136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077712.369710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077712.369711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077712.369713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077712.369716:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077712.369725:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077712.369729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077712/real 1769077712] req@ffff935002a61880 x1855008913515264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077712.369736:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913515264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077712.369741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077712.369742:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077712.369744:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077712.561185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077713.073194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077713.073195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077713.073195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077713.073249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077713.393187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077713.585125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077714.097123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077714.097125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077714.097134:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077714.097160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077714.417200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077714.609190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077715.121187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077715.121187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077715.121187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077715.121241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077715.441169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077715.633159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077716.145189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077716.145189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077716.145193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077716.145226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077716.465145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077716.657119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077717.169198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077717.169198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077717.169208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077717.169262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077717.489130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077717.489171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077717.489181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077717.489186:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077717.489197:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3423 00000100:00080000:2.0:1769077717.489209:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077717.489217:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077717.489259:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913515392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077717.489288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077717.489292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077717.489294:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077717.489299:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3423 00000100:00080000:2.0:1769077717.489303:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077717.489308:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077717.489320:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077717.489331:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913515392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077717.489336:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913515520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077717.489354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077717.489358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077717.489360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077717.489365:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3423 00000100:00080000:2.0:1769077717.489370:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077717.489375:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077717.489381:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077717.489394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913515648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077717.489405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077717.489409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077717.489411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077717.489416:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3423 00000100:00080000:2.0:1769077717.489422:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077717.489426:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077717.489432:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077717.489437:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913515520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077717.489449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913515776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077717.489463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077717.489467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077717.489469:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077717.489476:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3423 00000100:00100000:0.0:1769077717.489480:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913515648:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769077717.489484:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077717.489492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077717.489499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077717.489513:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913515776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077717.489515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913515904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077717.489534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077717.489538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077717.489540:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077717.489546:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3423 00000100:00100000:0.0:1769077717.489551:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515904:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077717.489552:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077717.489563:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077717.489570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077717.489586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913516032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077717.489600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077717.489603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077717.489605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077717.489610:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3423 00000100:00080000:2.0:1769077717.489615:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077717.489619:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077717.489625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077717.489628:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077717.489647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913516160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077717.489671:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077717.681100:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077718.193161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077718.193175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077718.193224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077718.193322:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077718.513235:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077718.705143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077719.217105:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077719.217126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077719.217132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077719.217214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077719.537215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077719.729167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077720.241130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077720.241130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077720.241136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077720.241252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077720.561340:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077720.561449:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077720.561506:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077720.561524:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077720.561569:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077720.561579:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077720.561589:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077720.561599:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077720.561608:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077720.561616:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077720.561746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077717/real 1769077720] req@ffff935002a62680 x1855008913516160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077720.561770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913516160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077720.561782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077720.561788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077720.561793:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077720.561802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077717/real 1769077720] req@ffff935002a61180 x1855008913516032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077720.561813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913516032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077720.561820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077720.561824:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077720.561827:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077720.561835:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077717/real 1769077720] req@ffff935002a61880 x1855008913515904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077720.561846:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913515904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077720.561855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077720.561859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077720.561863:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913515904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077720.561870:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077717/real 1769077720] req@ffff935002a62300 x1855008913515776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077720.561882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913515776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077720.561890:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077720.561895:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077720.561898:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913515776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077720.561905:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077717/real 1769077720] req@ffff935002a62a00 x1855008913515520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077720.561915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913515520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077720.561923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077720.561927:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077720.561930:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913515520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077720.561938:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077717/real 1769077720] req@ffff935002a61500 x1855008913515648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077720.561950:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913515648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077720.561958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077720.561962:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077720.561965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913515648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077720.561971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077717/real 1769077720] req@ffff935002a60000 x1855008913515392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077720.563219:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913515392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077720.563231:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077720.563236:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077720.563240:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913515392:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077720.753144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077721.265167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077721.265167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077721.265170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077721.265238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077721.585206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077721.777163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077722.289172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077722.289172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077722.289174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077722.289210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077722.609177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077722.609187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077722.609191:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077722.609201:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3428 00000100:00080000:3.0:1769077722.609212:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077722.609222:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077722.609232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077722.609261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913516288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077722.609295:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077722.609299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077722.609300:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077722.609305:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3428 00000100:00080000:3.0:1769077722.609308:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077722.609312:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077722.609323:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077722.609338:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913516416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077722.609349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077722.609352:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077722.609353:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077722.609357:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3428 00000100:00080000:3.0:1769077722.609359:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077722.609363:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077722.609367:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077722.609376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913516544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077722.609384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077722.609387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077722.609388:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077722.609392:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3428 00000100:00080000:3.0:1769077722.609394:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077722.609398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077722.609402:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077722.609412:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913516672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077722.609420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077722.609424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077722.609425:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077722.609429:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3428 00000100:00080000:3.0:1769077722.609432:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077722.609436:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077722.609440:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077722.609454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913516800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077722.609464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077722.609467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077722.609469:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077722.609472:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3428 00000100:00080000:3.0:1769077722.609475:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077722.609477:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077722.609482:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077722.609489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913516928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077722.609496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077722.609499:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077722.609501:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077722.609504:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3428 00000100:00080000:3.0:1769077722.609507:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077722.609510:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077722.609514:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077722.609524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913517056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077722.609657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913516288:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769077722.609749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077722.609805:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077722/real 1769077722] req@ffff935002a7d180 x1855008913516288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077722.609831:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913516288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077722.609841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077722.609846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077722.609850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913516288:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077722.609858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913516416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077722.609880:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077722.609895:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077722/real 1769077722] req@ffff935002a7dc00 x1855008913516416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077722.609906:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913516416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077722.609915:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077722.609918:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077722.609921:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913516416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077722.609927:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913516544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077722.609946:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077722.609956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077722/real 1769077722] req@ffff935002a7f800 x1855008913516544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077722.609998:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913516544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077722.610010:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077722.610013:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077722.610016:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913516544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077722.610021:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913516672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077722.610046:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077722.610089:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077722/real 1769077722] req@ffff935002a7c000 x1855008913516672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077722.610099:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913516672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077722.610106:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077722.610109:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077722.610111:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913516672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077722.610117:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516800:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077722.610134:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077722.610144:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077722/real 1769077722] req@ffff935002a7df80 x1855008913516800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077722.610154:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913516800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077722.610160:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077722.610163:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077722.610166:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077722.610170:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077722.610186:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077722.610196:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077722/real 1769077722] req@ffff935002a7fb80 x1855008913516928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077722.610206:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913516928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077722.610212:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077722.610215:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077722.610217:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913516928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077722.610222:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077722.610236:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077722.610246:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077722/real 1769077722] req@ffff935002a7f100 x1855008913517056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077722.610256:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913517056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077722.610264:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077722.610267:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077722.610270:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077722.801126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077723.313177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077723.313177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077723.313204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077723.313310:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077723.633242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077723.825179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077724.337149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077724.337149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077724.337176:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077724.337269:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077724.657182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077724.849147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077725.361163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077725.361168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077725.361197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077725.361200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077725.681182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077725.873167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077726.385149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077726.385152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077726.385189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077726.385298:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077726.705165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077726.897156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077727.409198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077727.409213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077727.409234:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077727.409241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077727.729143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077727.729149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077727.729151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077727.729155:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077727.729163:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3433 00000100:00080000:3.0:1769077727.729177:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077727.729183:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077727.729212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913517184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077727.729237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077727.729240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077727.729241:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077727.729244:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3433 00000100:00080000:3.0:1769077727.729246:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077727.729248:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077727.729255:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077727.729263:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913517312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077727.729270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077727.729272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077727.729274:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077727.729278:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3433 00000100:00080000:3.0:1769077727.729283:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077727.729287:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077727.729293:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077727.729306:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913517440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077727.729315:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077727.729321:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077727.729323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913517184:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077727.729325:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077727.729340:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3433 00000100:00080000:3.0:1769077727.729345:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077727.729348:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077727.729353:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077727.729366:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913517568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077727.729376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077727.729378:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077727.729380:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077727.729383:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3433 00000100:00080000:3.0:1769077727.729387:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077727.729390:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077727.729395:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077727.729407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913517696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077727.729413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077727.729417:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077727.729419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077727.729422:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077727.729426:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3433 00000100:00080000:3.0:1769077727.729428:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077727.729431:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077727.729434:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077727.729443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913517824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077727.729448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077727.729451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077727.729452:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077727.729455:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3433 00000100:00080000:3.0:1769077727.729457:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077727.729460:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077727.729464:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077727.729479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913517952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077727.729481:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077727/real 1769077727] req@ffff935002a7c700 x1855008913517184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077727.729514:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913517184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077727.729527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077727.729533:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077727.729537:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913517184:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077727.729545:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913517312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077727.729566:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077727.729577:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077727/real 1769077727] req@ffff935002a7e300 x1855008913517312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077727.729588:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913517312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077727.729597:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077727.729600:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077727.729603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913517312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077727.729630:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913517440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077727.729658:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077727.729668:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077727/real 1769077727] req@ffff935002a7ca80 x1855008913517440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077727.729679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913517440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077727.729687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077727.729690:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077727.729693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913517440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077727.729699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913517568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077727.729714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077727.729722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077727/real 1769077727] req@ffff935002a7c380 x1855008913517568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077727.729733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913517568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077727.729740:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077727.729743:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077727.729747:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913517568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077727.729752:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517696:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077727.729765:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077727.729773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077727/real 1769077727] req@ffff935002a7f100 x1855008913517696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077727.729790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913517696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077727.729797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077727.729800:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077727.729803:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517696:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077727.729809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077727.729822:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077727.729831:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077727/real 1769077727] req@ffff935002a7fb80 x1855008913517824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077727.729841:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913517824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077727.729849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077727.729851:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077727.729854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077727.729859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077727.729900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077727.729911:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077727/real 1769077727] req@ffff935002a7df80 x1855008913517952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077727.729922:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913517952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077727.729936:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077727.729938:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077727.729941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913517952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077727.921164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077728.433165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077728.433166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077728.433170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077728.433245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077728.753145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077728.945190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077729.457155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077729.457155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077729.457187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077729.457193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077729.777170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077729.969189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077730.481164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077730.481164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077730.481165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077730.481233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077730.801242:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077730.993107:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077731.505157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077731.505157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077731.505157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077731.505214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077731.825157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077732.017155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077732.529181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077732.529181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077732.529186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077732.529222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077732.849149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077732.849149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077732.849159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077732.849163:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077732.849171:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3438 00000100:00080000:1.0:1769077732.849181:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077732.849187:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077732.849221:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913518080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077732.849246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077732.849249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077732.849250:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077732.849253:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3438 00000100:00080000:1.0:1769077732.849256:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077732.849258:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077732.849266:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077732.849275:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913518208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077732.849282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077732.849285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077732.849287:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077732.849290:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3438 00000100:00080000:1.0:1769077732.849293:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077732.849298:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077732.849301:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913518080:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077732.849303:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077732.849316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913518336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077732.849324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077732.849327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077732.849328:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077732.849330:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3438 00000100:00080000:1.0:1769077732.849333:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077732.849335:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077732.849338:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077732.849346:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913518464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077732.849352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077732.849355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077732.849357:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077732.849359:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3438 00000100:00080000:1.0:1769077732.849361:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077732.849363:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077732.849366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077732.849371:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077732.849378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913518592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077732.849386:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077732.849389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077732.849390:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077732.849393:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3438 00000100:00080000:1.0:1769077732.849396:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077732.849398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077732.849401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077732.849409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913518720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077732.849414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077732.849417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077732.849419:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077732.849422:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077732/real 1769077732] req@ffff9350100ca680 x1855008913518080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077732.849425:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3438 00000100:00080000:1.0:1769077732.849431:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077732.849438:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077732.849443:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077732.849451:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913518080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077732.849454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913518848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077732.849479:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077732.849494:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077732.849498:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913518080:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077732.849504:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913518208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077732.849523:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077732.849537:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077732/real 1769077732] req@ffff9350100cad80 x1855008913518208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077732.849546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913518208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077732.849553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077732.849555:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077732.849558:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913518208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077732.849575:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913518336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077732.849587:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077732.849593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077732/real 1769077732] req@ffff9350100c9c00 x1855008913518336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077732.849600:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913518336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077732.849606:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077732.849609:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077732.849611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913518336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077732.849614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913518464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077732.849623:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077732.849628:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077732/real 1769077732] req@ffff9350100cb100 x1855008913518464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077732.849635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913518464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077732.849640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077732.849642:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077732.849644:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913518464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077732.849648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913518592:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077732.849658:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077732.849663:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077732/real 1769077732] req@ffff9350100c9180 x1855008913518592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077732.849670:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913518592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077732.849675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077732.849677:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077732.849679:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913518592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077732.849682:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913518720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077732.849692:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077732.849697:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077732/real 1769077732] req@ffff9350100c8a80 x1855008913518720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077732.849704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913518720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077732.849709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077732.849711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077732.849713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913518720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077732.849716:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913518848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077732.849724:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077732.849731:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077732/real 1769077732] req@ffff9350100c8000 x1855008913518848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077732.849738:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913518848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077732.849743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077732.849745:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077732.849747:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913518848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077733.041152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077733.553128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077733.553144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077733.553176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077733.553201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077733.873218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077734.065133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077734.577167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077734.577167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077734.577177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077734.577287:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077734.897206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077735.089117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077735.601208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077735.601208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077735.601208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077735.601245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077735.921195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077736.113188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077736.625177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077736.625177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077736.625179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077736.625213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077736.945123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077737.137192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077737.649199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077737.649199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077737.649205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077737.649245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077737.969199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077737.969203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077737.969212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077737.969217:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.969228:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3443 00000100:00080000:0.0:1769077737.969240:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.969247:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077737.969288:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913518976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.969321:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.969324:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077737.969325:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.969328:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3443 00000100:00080000:0.0:1769077737.969330:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.969334:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077737.969344:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077737.969353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913519104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.969362:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.969365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077737.969366:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077737.969371:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913518976:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077737.969372:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3443 00000100:00080000:0.0:1769077737.969376:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.969382:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077737.969386:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077737.969393:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913519232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.969399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.969402:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077737.969403:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.969405:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3443 00000100:00080000:0.0:1769077737.969407:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.969410:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077737.969413:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077737.969420:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913519360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.969426:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.969429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077737.969430:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.969433:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3443 00000100:00080000:0.0:1769077737.969436:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.969439:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077737.969442:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077737.969449:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077737.969451:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913519488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.969458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.969461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077737.969462:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.969465:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3443 00000100:00080000:0.0:1769077737.969468:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.969470:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077737.969474:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077737.969482:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913519616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.969488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.969490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077737.969491:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.969494:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3443 00000100:00080000:0.0:1769077737.969496:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077737.969498:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9350103f9180 x1855008913518976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077737.969502:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077737.969509:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077737.969521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913519744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077737.969525:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913518976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077737.969543:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077737.969549:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077737.969555:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913518976:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077737.969584:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913519104:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077737.969608:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077737.969631:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9350103f9f80 x1855008913519104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077737.969641:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913519104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077737.969648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077737.969650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077737.969653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913519104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077737.969657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913519232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077737.969671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077737.969678:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9350103f9c00 x1855008913519232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077737.969686:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913519232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077737.969692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077737.969694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077737.969696:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913519232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077737.969700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913519360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077737.969738:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077737.969746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9350103fbb80 x1855008913519360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077737.969755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913519360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077737.969761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077737.969763:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077737.969766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913519360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077737.969769:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913519488:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077737.969781:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077737.969786:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9350103f9500 x1855008913519488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077737.969794:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913519488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077737.969800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077737.969802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077737.969804:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913519488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077737.969807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913519616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077737.969816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077737.969821:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9350103f8380 x1855008913519616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077737.969829:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913519616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077737.969834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077737.969836:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077737.969838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913519616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077737.969841:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913519744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077737.969849:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077737.969855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9350103fb800 x1855008913519744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077737.969861:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913519744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077737.969866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077737.969868:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077737.969871:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913519744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077738.161168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077738.673206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077738.673207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077738.673209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077738.673253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077738.993139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077739.185120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077739.697145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077739.697145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077739.697177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077739.697205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077740.017158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077740.209161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077740.721165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077740.721191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077740.721193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077740.721239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077741.041143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077741.233120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077741.745152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077741.745153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077741.745183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077741.745222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077742.065152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077742.257136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077742.769176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077742.769209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077742.769210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077742.769264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077743.089175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077743.089181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077743.089188:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077743.089192:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077743.089209:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3448 00000100:00080000:3.0:1769077743.089220:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077743.089227:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077743.089261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913519872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077743.089286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077743.089289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077743.089290:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077743.089298:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3448 00000100:00080000:3.0:1769077743.089301:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077743.089304:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077743.089312:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077743.089321:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913520000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077743.089330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077743.089334:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077743.089337:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077743.089339:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913519872:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077743.089342:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3448 00000100:00080000:3.0:1769077743.089349:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077743.089351:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077743.089355:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077743.089365:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913520128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077743.089372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077743.089375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077743.089377:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077743.089379:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3448 00000100:00080000:3.0:1769077743.089395:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077743.089398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077743.089401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077743.089409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913520256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077743.089416:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077743.089418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077743.089422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077743.089424:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077743.089428:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3448 00000100:00080000:3.0:1769077743.089431:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077743.089434:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077743.089438:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077743.089448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913520384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077743.089456:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077743/real 1769077743] req@ffff935002a7c000 x1855008913519872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077743.089458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077743.089466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077743.089470:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077743.089478:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3448 00000100:00080000:3.0:1769077743.089482:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077743.089485:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913519872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077743.089486:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077743.089493:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077743.089502:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077743.089507:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913520512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077743.089509:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077743.089517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913519872:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077743.089524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077743.089528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077743.089530:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077743.089534:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3448 00000100:00080000:3.0:1769077743.089536:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077743.089538:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077743.089542:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077743.089545:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077743.089552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913520640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077743.089567:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077743.089578:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077743/real 1769077743] req@ffff935002a7f800 x1855008913520000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077743.089587:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913520000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077743.089593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077743.089596:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077743.089598:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077743.089602:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077743.089614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077743.089641:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077743/real 1769077743] req@ffff935002a7dc00 x1855008913520128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077743.089650:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913520128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077743.089655:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077743.089658:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077743.089660:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077743.089669:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077743.089683:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077743.089689:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077743/real 1769077743] req@ffff935002a7d180 x1855008913520256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077743.089697:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913520256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077743.089703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077743.089705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077743.089707:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077743.089710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913520384:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077743.089719:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077743.089725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077743/real 1769077743] req@ffff935002a7df80 x1855008913520384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077743.089736:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913520384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077743.089741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077743.089743:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077743.089745:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913520384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077743.089748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913520512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077743.089757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077743.089763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077743/real 1769077743] req@ffff935002a7fb80 x1855008913520512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077743.089775:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913520512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077743.089783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077743.089787:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077743.089790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913520512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077743.089811:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913520640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077743.089832:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077743.089841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077743/real 1769077743] req@ffff935002a7f100 x1855008913520640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077743.089849:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913520640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077743.089862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077743.089866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077743.089869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913520640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077743.281164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077743.793122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077743.793130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077743.793143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077743.793171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077744.113166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077744.305155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077744.817124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077744.817128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077744.817139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077744.817156:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077745.137174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077745.329210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077745.841113:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077745.841113:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077745.841134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077745.841162:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077746.161139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077746.353203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077746.865125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077746.865125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077746.865154:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077746.865169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077747.185175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077747.377160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077747.889173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077747.889173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077747.889206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077747.889210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077748.209144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077748.209158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1769077748.209163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077748.209164:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077748.209177:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3453 00000100:00080000:3.0:1769077748.209188:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077748.209197:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077748.209236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913520768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077748.209262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077748.209265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077748.209267:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077748.209270:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3453 00000100:00080000:3.0:1769077748.209273:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077748.209277:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077748.209285:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077748.209299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913520896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077748.209304:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913520768:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077748.209327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077748.209332:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077748.209335:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077748.209341:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3453 00000100:00080000:3.0:1769077748.209367:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077748.209373:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077748.209379:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077748.209387:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077748.209404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913521024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077748.209413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077748.209417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077748.209419:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077748.209424:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077748/real 1769077748] req@ffff935002a7c380 x1855008913520768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077748.209427:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3453 00000100:00080000:3.0:1769077748.209435:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077748.209442:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077748.209447:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913520768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077748.209450:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1769077748.209461:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077748.209465:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077748.209471:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913520768:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077748.209472:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913521152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077748.209490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077748.209495:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520896:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077748.209496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077748.209503:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077748.209509:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3453 00000100:00080000:3.0:1769077748.209519:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077748.209525:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077748.209529:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077748.209558:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077748/real 1769077748] req@ffff935002a7ca80 x1855008913520896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077748.209563:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077748.209577:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913520896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077748.209584:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913521280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077748.209611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077748.209614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077748.209615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077748.209617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077748.209624:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077748.209624:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3453 00000100:00080000:3.0:1769077748.209631:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077748.209636:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913520896:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077748.209637:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077748.209644:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077748.209651:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077748.209656:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913521408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077748.209668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077748.209672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077748.209674:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077748.209682:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077748.209682:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3453 00000100:00080000:3.0:1769077748.209690:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077748.209699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077748/real 1769077748] req@ffff935002a7e300 x1855008913521024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077748.209699:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077748.209710:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077748.209714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913521024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077748.209722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077748.209727:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077748.209727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913521536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077748.209732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077748.209743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077748.209764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077748.209775:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077748/real 1769077748] req@ffff935002a7c700 x1855008913521152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077748.209783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913521152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077748.209788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077748.209790:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077748.209792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077748.209804:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913521280:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077748.209824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077748.209841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077748/real 1769077748] req@ffff935002a7c380 x1855008913521280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077748.209854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913521280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077748.209863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077748.209866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077748.209868:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913521280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077748.209873:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913521408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077748.209888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077748.209895:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077748/real 1769077748] req@ffff935002a7f100 x1855008913521408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077748.209903:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913521408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077748.209909:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077748.209911:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077748.209913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913521408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077748.209916:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913521536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077748.209925:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077748.209931:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077748/real 1769077748] req@ffff935002a7fb80 x1855008913521536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077748.209938:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913521536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077748.209943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077748.209944:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077748.209946:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913521536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077748.401136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077748.913148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077748.913149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077748.913162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077748.913257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077749.233113:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077749.425137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077749.937172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077749.937173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077749.937182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077749.937206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077750.257156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077750.449126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077750.961120:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077750.961132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077750.961154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077750.961191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077751.281152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077751.473145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077751.985157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077751.985157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077751.985159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077751.985190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077752.305166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077752.497204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077753.009129:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077753.009140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077753.009143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077753.009185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077753.329162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077753.329195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077753.329201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077753.329205:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077753.329215:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3459 00000100:00080000:1.0:1769077753.329226:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077753.329233:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077753.329270:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913521664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077753.329299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077753.329303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077753.329305:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077753.329309:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3459 00000100:00080000:1.0:1769077753.329313:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077753.329318:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077753.329328:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077753.329346:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913521792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077753.329346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913521664:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077753.329365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077753.329368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077753.329370:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077753.329374:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3459 00000100:00080000:1.0:1769077753.329379:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077753.329383:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077753.329389:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077753.329401:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913521920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077753.329412:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077753.329415:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077753.329417:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077753.329422:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3459 00000800:00000100:2.0:1769077753.329428:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077753.329430:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077753.329437:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077753.329442:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077753.329471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913522048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077753.329483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077753.329487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077753.329489:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077753.329493:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3459 00000100:00080000:1.0:1769077753.329498:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077753.329502:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077753.329508:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077753.329518:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077753/real 1769077753] req@ffff9350100c9f80 x1855008913521664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077753.329523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913522176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077753.329550:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913521664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077753.329575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077753.329577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077753.329583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077753.329585:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077753.329587:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077753.329592:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913521664:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077753.329595:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3459 00000100:00080000:1.0:1769077753.329601:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077753.329604:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077753.329609:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077753.329623:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913522304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077753.329623:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521792:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077753.329637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077753.329641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077753.329643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077753.329646:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3459 00000100:00080000:1.0:1769077753.329649:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077753.329655:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077753.329657:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769077753.329659:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077753.329670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913522432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077753.329673:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077753/real 1769077753] req@ffff9350100c8000 x1855008913521792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077753.329700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913521792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077753.329707:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077753.329710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077753.329713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077753.329718:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077753.329733:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077753.329739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077753/real 1769077753] req@ffff9350100c8a80 x1855008913521920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077753.329747:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913521920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077753.329753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077753.329755:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077753.329758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913521920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077753.329764:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077753.329796:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077753.329808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077753/real 1769077753] req@ffff9350100c9180 x1855008913522048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077753.329821:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913522048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077753.329834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077753.329837:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077753.329839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077753.329843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913522176:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077753.329858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077753.329865:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077753/real 1769077753] req@ffff9350100cb100 x1855008913522176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077753.329873:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913522176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077753.329879:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077753.329881:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077753.329883:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913522176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077753.329895:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913522304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077753.329906:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077753.329916:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077753/real 1769077753] req@ffff9350100c9c00 x1855008913522304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077753.329928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913522304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077753.329936:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077753.329939:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077753.329943:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913522304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077753.329950:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913522432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077753.329970:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077753.329979:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077753/real 1769077753] req@ffff9350100cad80 x1855008913522432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077753.329987:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913522432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077753.329992:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077753.329994:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077753.329996:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913522432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077753.521147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077754.033153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077754.033153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077754.033156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077754.033187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077754.353168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077754.545208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077755.057153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077755.057153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077755.057168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077755.057201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077755.377164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077755.569122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077756.081154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077756.081154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077756.081160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077756.081187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077756.401157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077756.593190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077757.105157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077757.105157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077757.105162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077757.105248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077757.425186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077757.617161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077758.129238:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077758.129239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077758.129244:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077758.129297:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077758.449190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077758.449208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077758.449218:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077758.449236:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077758.449249:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3464 00000100:00080000:3.0:1769077758.449263:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077758.449272:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077758.449322:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913522560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077758.449357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077758.449378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077758.449381:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077758.449387:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3464 00000100:00080000:3.0:1769077758.449392:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077758.449399:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077758.449428:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077758.449439:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913522560:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077758.449446:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913522688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077758.449459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077758.449463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077758.449465:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077758.449470:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3464 00000100:00080000:3.0:1769077758.449475:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077758.449479:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077758.449486:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077758.449498:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913522816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077758.449508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077758.449512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077758.449514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077758.449520:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077758.449521:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3464 00000100:00080000:3.0:1769077758.449528:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077758.449533:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077758.449539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077758.449552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913522944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077758.449562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077758.449567:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077758/real 1769077758] req@ffff935002a7df80 x1855008913522560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077758.449568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077758.449572:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077758.449582:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3464 00000100:00080000:3.0:1769077758.449588:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077758.449593:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913522560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077758.449595:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077758.449606:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077758.449609:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077758.449616:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077758.449620:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913522560:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077758.449624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913523072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077758.449638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077758.449641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077758.449643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077758.449647:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3464 00000100:00080000:3.0:1769077758.449651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077758.449652:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522688:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077758.449660:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077758.449667:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077758.449674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077758.449683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077758/real 1769077758] req@ffff935002a7d180 x1855008913522688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077758.449694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913523200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077758.449696:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913522688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077758.449725:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077758.449729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077758.449732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077758.449737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522816:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077758.449739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077758.449744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077758.449747:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077758.449751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077758.449754:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3464 00000100:00080000:3.0:1769077758.449761:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077758.449762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077758/real 1769077758] req@ffff935002a7dc00 x1855008913522816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077758.449771:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077758.449778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913522816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077758.449780:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077758.449789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077758.449793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077758.449795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077758.449801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077758.449801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913523328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077758.449821:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077758.449830:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077758/real 1769077758] req@ffff935002a7f800 x1855008913522944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077758.449841:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913522944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077758.449847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077758.449849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077758.449851:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913522944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077758.449854:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523072:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077758.449864:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077758.449869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077758/real 1769077758] req@ffff935002a7c000 x1855008913523072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077758.449877:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913523072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077758.449882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077758.449884:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077758.449886:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523072:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077758.449918:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077758.449932:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077758.449939:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077758/real 1769077758] req@ffff935002a7df80 x1855008913523200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077758.449948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913523200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077758.449959:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077758.449962:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077758.449964:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077758.449968:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077758.449978:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077758.449984:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077758/real 1769077758] req@ffff935002a7fb80 x1855008913523328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077758.449993:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913523328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077758.449999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077758.450001:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077758.450003:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077758.641180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077759.153157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077759.153157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077759.153160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077759.153220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077759.473187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077759.665145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077760.177173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077760.177173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077760.177173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077760.177214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077760.497170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077760.689133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077761.201204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077761.201205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077761.201218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077761.201258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077761.521215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077761.713168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077762.225152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077762.225172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077762.225176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077762.225213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077762.545136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077762.737175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077763.249150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077763.249150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077763.249158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077763.249195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077763.569168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077763.569174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077763.569183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077763.569188:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077763.569198:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3469 00000100:00080000:0.0:1769077763.569210:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077763.569219:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077763.569262:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913523456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077763.569322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077763.569326:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077763.569328:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077763.569333:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3469 00000100:00080000:0.0:1769077763.569336:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077763.569340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077763.569350:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077763.569365:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913523584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077763.569377:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913523456:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077763.569378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077763.569387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077763.569389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077763.569394:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3469 00000100:00080000:0.0:1769077763.569397:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077763.569401:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077763.569407:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077763.569424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913523712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077763.569434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077763.569438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077763.569440:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077763.569444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3469 00000800:00000100:2.0:1769077763.569448:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077763.569453:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077763.569460:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077763.569467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077763.569481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913523840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077763.569491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077763.569495:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077763/real 1769077763] req@ffff9350103f8700 x1855008913523456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077763.569497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077763.569500:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077763.569511:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3469 00000100:00080000:0.0:1769077763.569516:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077763.569523:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077763.569523:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913523456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077763.569531:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077763.569538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077763.569546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913523968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077763.569546:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077763.569554:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913523456:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077763.569565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077763.569568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077763.569569:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077763.569573:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3469 00000100:00080000:0.0:1769077763.569576:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077763.569582:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077763.569582:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913523584:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769077763.569590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077763.569600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913524096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077763.569608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077763.569611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077763.569612:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077763.569616:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077763.569620:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3469 00000100:00080000:0.0:1769077763.569625:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077763.569628:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077763.569631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077763.569638:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077763/real 1769077763] req@ffff9350103fb100 x1855008913523584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077763.569642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913524224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077763.569680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913523584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077763.569690:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077763.569693:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077763.569696:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913523584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077763.569701:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913523712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077763.569718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077763.569727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077763/real 1769077763] req@ffff9350103faa00 x1855008913523712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077763.569737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913523712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077763.569746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077763.569749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077763.569752:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913523712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077763.569759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913523840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077763.569780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077763.569791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077763/real 1769077763] req@ffff9350103fb480 x1855008913523840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077763.569803:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913523840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077763.569809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077763.569811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077763.569813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913523840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077763.569817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523968:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077763.569831:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077763.569837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077763/real 1769077763] req@ffff9350103fad80 x1855008913523968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077763.569845:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913523968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077763.569850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077763.569852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077763.569854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913523968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077763.569872:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077763.569891:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077763.569903:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077763/real 1769077763] req@ffff9350103f8700 x1855008913524096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077763.569923:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913524096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077763.569931:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077763.569934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077763.569938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077763.569944:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077763.569965:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077763.569977:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077763/real 1769077763] req@ffff9350103fb800 x1855008913524224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077763.569989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913524224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077763.569995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077763.569997:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077763.569999:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077763.761148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077764.273137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077764.273138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077764.273138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077764.273192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077764.593151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077764.785169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077765.297128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077765.297135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077765.297152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077765.297182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077765.617147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077765.809161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077766.321156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077766.321156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077766.321179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077766.321214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077766.641158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077766.833156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077767.345161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077767.345161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077767.345186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077767.345229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077767.665147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077767.857156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077768.369150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077768.369150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077768.369157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077768.369288:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077768.689160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077768.689163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077768.689173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077768.689178:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077768.689203:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3474 00000100:00080000:0.0:1769077768.689217:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077768.689225:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077768.689267:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913524352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077768.689306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077768.689310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077768.689312:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077768.689317:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3474 00000100:00080000:0.0:1769077768.689321:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077768.689325:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077768.689337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077768.689352:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913524480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077768.689360:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913524352:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077768.689368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077768.689374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077768.689376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077768.689381:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3474 00000100:00080000:0.0:1769077768.689386:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077768.689390:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077768.689396:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077768.689409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913524608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077768.689427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077768.689430:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077768.689435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077768.689439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077768.689444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3474 00000100:00080000:0.0:1769077768.689449:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077768.689454:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077768.689460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077768.689472:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077768/real 1769077768] req@ffff9350103f8380 x1855008913524352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077768.689477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913524736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077768.689497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077768.689502:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913524352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077768.689503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077768.689508:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077768.689518:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3474 00000100:00080000:0.0:1769077768.689526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077768.689526:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077768.689535:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077768.689537:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:0.0:1769077768.689544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077768.689545:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913524352:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077768.689565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913524864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077768.689573:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913524480:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077768.689587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077768.689592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077768.689595:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077768.689603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3474 00000800:00000100:2.0:1769077768.689603:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077768.689612:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077768.689620:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077768.689621:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077768/real 1769077768] req@ffff9350103f9500 x1855008913524480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077768.689631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077768.689635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913524480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077768.689643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077768.689650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913524992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077768.689657:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077768.689663:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913524480:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077768.689670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077768.689672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913524608:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077768.689680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077768.689684:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077768.689689:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3474 00000100:00080000:0.0:1769077768.689694:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077768.689695:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077768.689704:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077768.689708:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077768/real 1769077768] req@ffff9350103fbb80 x1855008913524608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077768.689713:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077768.689719:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913524608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077768.689727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077768.689732:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077768.689733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913525120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077768.689735:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913524608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077768.689746:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913524736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077768.689766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077768.689774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077768/real 1769077768] req@ffff9350103f9c00 x1855008913524736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077768.689782:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913524736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077768.689788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077768.689790:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077768.689792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913524736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077768.689803:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077768.689813:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077768.689818:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077768/real 1769077768] req@ffff9350103f9f80 x1855008913524864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077768.689826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913524864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077768.689831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077768.689840:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077768.689842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077768.689851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077768.689871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077768.689883:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077768/real 1769077768] req@ffff9350103f9180 x1855008913524992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077768.689897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913524992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077768.689906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077768.689909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077768.689913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913524992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077768.689920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913525120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077768.689940:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077768.689959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077768/real 1769077768] req@ffff9350103f8380 x1855008913525120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077768.689967:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913525120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077768.689973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077768.689975:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077768.689977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913525120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077768.881108:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077769.393161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077769.393167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077769.393197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077769.393215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077769.713152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077769.905130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077770.417187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077770.417187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077770.417187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077770.417250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077770.737208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077770.929243:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077771.441167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077771.441167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077771.441168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077771.441201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077771.761180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077771.953143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077772.465177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077772.465178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077772.465178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077772.465210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077772.785184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077772.977192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077773.489177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077773.489177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077773.489178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077773.489209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077773.809171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077773.809179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077773.809181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077773.809188:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077773.809197:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3479 00000100:00080000:0.0:1769077773.809208:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077773.809214:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077773.809248:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913525248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077773.809273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077773.809276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077773.809277:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077773.809280:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3479 00000100:00080000:0.0:1769077773.809282:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077773.809285:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077773.809292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077773.809301:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913525376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077773.809308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077773.809311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077773.809312:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077773.809315:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3479 00000100:00080000:0.0:1769077773.809317:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077773.809319:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077773.809324:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913525248:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769077773.809325:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077773.809338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913525504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077773.809346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077773.809348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077773.809350:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077773.809353:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3479 00000100:00080000:0.0:1769077773.809355:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077773.809357:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077773.809360:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077773.809368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913525632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077773.809374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077773.809376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077773.809378:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077773.809380:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3479 00000100:00080000:0.0:1769077773.809383:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077773.809385:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077773.809388:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077773.809391:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077773.809400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913525760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077773.809407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077773.809409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077773.809410:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077773.809413:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3479 00000100:00080000:0.0:1769077773.809415:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077773.809417:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077773.809420:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077773.809429:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913525888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077773.809435:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077773/real 1769077773] req@ffff9350103fb800 x1855008913525248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077773.809437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077773.809443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077773.809446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077773.809453:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3479 00000100:00080000:0.0:1769077773.809459:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077773.809461:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077773.809465:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077773.809472:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913525248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077773.809475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913526016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077773.809495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077773.809501:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077773.809504:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913525248:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077773.809511:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913525376:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077773.809531:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077773.809541:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077773/real 1769077773] req@ffff9350103f8700 x1855008913525376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077773.809550:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913525376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077773.809557:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077773.809559:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077773.809561:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913525376:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077773.809580:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913525504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077773.809590:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077773.809597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077773/real 1769077773] req@ffff9350103fad80 x1855008913525504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077773.809609:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913525504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077773.809623:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077773.809627:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077773.809631:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913525504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077773.809638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913525632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077773.809658:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077773.809680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077773/real 1769077773] req@ffff9350103fb480 x1855008913525632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077773.809692:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913525632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077773.809701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077773.809704:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077773.809707:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913525632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077773.809713:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913525760:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077773.809730:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077773.809739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077773/real 1769077773] req@ffff9350103faa00 x1855008913525760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077773.809750:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913525760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077773.809758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077773.809761:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077773.809764:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913525760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077773.809770:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913525888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077773.809785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077773.809794:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077773/real 1769077773] req@ffff9350103fb100 x1855008913525888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077773.809813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913525888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077773.809821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077773.809824:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077773.809827:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913525888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077773.809833:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077773.809852:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077773.809863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077773/real 1769077773] req@ffff9350103f8380 x1855008913526016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077773.809871:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913526016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077773.809877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077773.809879:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077773.809881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077774.001189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077774.513195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077774.513195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077774.513195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077774.513258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077774.833199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077775.025219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077775.537174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077775.537174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077775.537174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077775.537267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077775.857170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077776.049174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077776.561188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077776.561189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077776.561231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077776.561314:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077776.881178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077777.073146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077777.585161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077777.585161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077777.585182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077777.585186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077777.905157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077778.097206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077778.609152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077778.609152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077778.609152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077778.609195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077778.929170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077778.929182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077778.929183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077778.929189:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077778.929200:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3484 00000100:00080000:0.0:1769077778.929222:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077778.929232:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077778.929272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913526144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077778.929306:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077778.929310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077778.929318:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077778.929323:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3484 00000100:00080000:0.0:1769077778.929327:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077778.929332:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077778.929343:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077778.929357:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913526272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077778.929364:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913526144:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077778.929376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077778.929380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077778.929383:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077778.929389:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3484 00000100:00080000:0.0:1769077778.929393:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077778.929398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077778.929404:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077778.929418:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913526400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077778.929429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077778.929433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077778.929435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077778.929441:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077778.929445:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3484 00000100:00080000:0.0:1769077778.929451:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077778.929456:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077778.929461:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077778.929483:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913526528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077778.929489:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff9350103f9180 x1855008913526144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077778.929505:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077778.929511:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077778.929513:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077778.929521:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3484 00000100:00100000:2.0:1769077778.929522:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913526144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077778.929525:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077778.929533:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077778.929542:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077778.929546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077778.929554:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077778.929559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913526656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077778.929562:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913526144:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077778.929580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077778.929584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077778.929586:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077778.929591:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3484 00000100:00080000:0.0:1769077778.929596:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077778.929597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913526272:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077778.929603:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077778.929611:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077778.929627:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913526784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077778.929638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077778.929643:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077778.929645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077778.929649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077778.929656:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3484 00000100:00080000:0.0:1769077778.929661:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077778.929666:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff9350103f9f80 x1855008913526272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077778.929669:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077778.929678:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077778.929690:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913526272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077778.929695:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913526912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077778.929720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077778.929726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077778.929730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913526272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077778.929738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913526400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077778.929765:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077778.929779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff9350103f9c00 x1855008913526400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077778.929793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913526400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077778.929802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077778.929806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077778.929809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913526400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077778.929816:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913526528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077778.929836:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077778.929848:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff9350103fbb80 x1855008913526528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077778.929862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913526528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077778.929872:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077778.929875:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077778.929879:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913526528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077778.929886:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526656:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077778.929907:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077778.929919:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff9350103f9500 x1855008913526656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077778.929939:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913526656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077778.929949:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077778.929952:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077778.929956:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077778.929978:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077778.930000:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077778.930012:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff9350103f9180 x1855008913526784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077778.930083:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913526784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077778.930091:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077778.930095:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077778.930098:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077778.930105:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077778.930126:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077778.930140:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff9350103f8380 x1855008913526912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077778.930153:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913526912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077778.930169:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077778.930172:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077778.930176:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913526912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077779.121123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077779.633192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077779.633192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077779.633192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077779.633253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077779.953124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077780.145176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077780.657149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077780.657149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077780.657149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077780.657184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077780.977152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077781.169111:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077781.681141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077781.681174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077781.681183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077781.681184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077782.001124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077782.193133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077782.705139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077782.705139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077782.705147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077782.705206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077783.025179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077783.217171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077783.729156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077783.729195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077783.729199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077783.729211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077784.049174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077784.049184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077784.049188:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077784.049197:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077784.049213:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3489 00000100:00080000:2.0:1769077784.049227:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077784.049236:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077784.049282:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913527040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077784.049311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077784.049315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077784.049316:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077784.049320:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3489 00000100:00080000:2.0:1769077784.049324:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077784.049327:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077784.049337:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077784.049345:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913527040:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077784.049351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913527168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077784.049360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077784.049363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077784.049364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077784.049368:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3489 00000100:00080000:2.0:1769077784.049370:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077784.049372:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077784.049376:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077784.049384:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913527296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077784.049390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077784.049393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077784.049394:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077784.049396:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3489 00000100:00080000:2.0:1769077784.049401:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077784.049403:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077784.049406:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077784.049413:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913527424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077784.049420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077784.049423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077784.049424:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077784.049427:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3489 00000100:00080000:2.0:1769077784.049431:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077784.049438:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077784.049443:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077784.049454:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913527168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077784.049456:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913527552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077784.049470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077784.049475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077784.049480:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077784.049492:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913527296:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769077784.049497:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3489 00000100:00080000:2.0:1769077784.049504:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077784.049510:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077784.049516:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077784.049539:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913527680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077784.049540:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913527424:192.168.204.121@tcp:8:kworker.0 00000100:00080000:2.0:1769077784.049555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077784.049559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077784.049560:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077784.049564:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3489 00000100:00080000:2.0:1769077784.049568:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077784.049572:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077784.049577:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077784.049587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913527552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077784.049588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913527808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077784.049634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913527680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077784.049664:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913527808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077784.241139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077784.753122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077784.753122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077784.753133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077784.753196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077785.073179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077785.265148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077785.777170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077785.777170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077785.777177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077785.777284:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077786.097162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077786.289223:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077786.801149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077786.801155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077786.801163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077786.801191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077787.121125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077787.121301:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077787.121337:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077787.121354:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077787.121396:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077787.121408:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077787.121421:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077787.121433:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077787.121441:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077787.121448:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077787.121521:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077784/real 1769077787] req@ffff935002a62a00 x1855008913527808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077787.121547:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913527808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077787.121558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077787.121564:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077787.121569:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913527808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077787.121578:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077784/real 1769077787] req@ffff935002a60000 x1855008913527552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077787.121592:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913527552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077787.121602:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077787.121607:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077787.121610:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913527552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077787.121618:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077784/real 1769077787] req@ffff935002a61500 x1855008913527680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077787.121632:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913527680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077787.121641:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077787.121644:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077787.121648:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913527680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077787.121660:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077784/real 1769077787] req@ffff935002a63100 x1855008913527168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077787.121676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913527168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077787.121685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077787.121690:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077787.121694:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913527168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077787.121704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077784/real 1769077787] req@ffff935002a61c00 x1855008913527296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077787.121718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913527296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077787.121727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077787.121731:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077787.121735:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913527296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077787.121743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077784/real 1769077787] req@ffff935002a61f80 x1855008913527424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077787.121769:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913527424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077787.121779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077787.121783:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077787.121787:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913527424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077787.121796:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077784/real 1769077787] req@ffff935002a60a80 x1855008913527040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077787.121811:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913527040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077787.121819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077787.121823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077787.121827:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913527040:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077787.313118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077787.825128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077787.825129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077787.825130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077787.825183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077788.145161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077788.337133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077788.849153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077788.849153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077788.849154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077788.849208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077789.169163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077789.169200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077789.169207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077789.169211:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077789.169220:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3494 00000100:00080000:3.0:1769077789.169233:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077789.169241:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077789.169273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913527936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077789.169291:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077789.169294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077789.169296:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077789.169300:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3494 00000100:00080000:3.0:1769077789.169304:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077789.169307:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077789.169317:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077789.169326:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913527936:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077789.169334:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913528064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077789.169346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077789.169351:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077789.169353:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077789.169358:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3494 00000100:00080000:3.0:1769077789.169362:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077789.169366:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077789.169372:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077789.169384:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913528192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077789.169395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077789.169399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769077789.169404:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077789.169404:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077789.169413:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3494 00000100:00080000:3.0:1769077789.169418:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077789.169423:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077789.169428:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077789.169441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913528320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077789.169451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077789.169455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077789.169457:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077789.169463:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3494 00000100:00080000:3.0:1769077789.169468:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077789.169472:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077789/real 1769077789] req@ffff935002a7f100 x1855008913527936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077789.169476:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077789.169486:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077789.169503:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913528448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077789.169509:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913527936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077789.169528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077789.169533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077789.169534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077789.169540:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077789.169544:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077789.169549:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3494 00000100:00100000:0.0:1769077789.169554:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913527936:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077789.169557:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077789.169567:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077789.169575:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077789.169594:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913528576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077789.169596:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528064:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077789.169614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077789.169619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769077789.169632:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077789.169634:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077789.169642:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3494 00000100:00080000:3.0:1769077789.169647:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077789.169653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077789/real 1769077789] req@ffff935002a7c380 x1855008913528064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077789.169654:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077789.169665:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077789.169674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913528064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077789.169683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913528704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077789.169687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077789.169696:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077789.169704:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077789.169714:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077789.169736:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077789.169748:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077789/real 1769077789] req@ffff935002a7c700 x1855008913528192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077789.169758:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913528192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077789.169766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077789.169769:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077789.169772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077789.169778:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077789.169796:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077789.169807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077789/real 1769077789] req@ffff935002a7e300 x1855008913528320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077789.169818:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913528320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077789.169828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077789.169831:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077789.169835:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077789.169858:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913528448:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077789.169880:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077789.169892:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077789/real 1769077789] req@ffff935002a7ca80 x1855008913528448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077789.169904:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913528448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077789.169913:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077789.169917:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077789.169920:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913528448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077789.169943:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913528576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077789.169964:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077789.169979:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077789/real 1769077789] req@ffff935002a7fb80 x1855008913528576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077789.169996:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913528576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077789.170005:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077789.170009:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077789.170014:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913528576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077789.170020:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913528704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077789.170042:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077789.170114:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077789/real 1769077789] req@ffff935002a7df80 x1855008913528704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077789.170130:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913528704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077789.170141:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077789.170144:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077789.170148:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913528704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077789.361151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077789.873162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077789.873162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077789.873191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077789.873205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077790.193171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077790.385147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077790.897172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077790.897172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077790.897175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077790.897207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077791.217157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077791.409170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077791.921144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077791.921190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077791.921199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077791.921205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077792.241115:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077792.433146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077792.945160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077792.945165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077792.945171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077792.945203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077793.265141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077793.457141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077793.969134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077793.969134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077793.969143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077793.969157:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077794.289135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077794.289140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077794.289149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077794.289154:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077794.289164:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3499 00000100:00080000:3.0:1769077794.289177:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077794.289184:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077794.289222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913528832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077794.289248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077794.289251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077794.289253:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077794.289256:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3499 00000100:00080000:3.0:1769077794.289259:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077794.289270:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077794.289281:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077794.289287:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913528832:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077794.289294:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913528960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077794.289303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077794.289306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077794.289308:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077794.289311:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3499 00000100:00080000:3.0:1769077794.289314:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077794.289316:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077794.289319:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077794.289329:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913529088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077794.289336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077794.289338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077794.289340:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077794.289342:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3499 00000100:00080000:3.0:1769077794.289345:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077794.289347:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077794.289350:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077794.289354:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077794.289362:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913529216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077794.289370:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077794.289373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077794.289374:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077794.289377:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3499 00000100:00080000:3.0:1769077794.289379:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077794.289381:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077794.289385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077794.289393:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077794/real 1769077794] req@ffff935002a7c000 x1855008913528832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077794.289397:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913529344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077794.289424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077794.289427:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913528832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077794.289432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077794.289438:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077794.289441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077794.289449:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077794.289449:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3499 00000100:00100000:0.0:1769077794.289456:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913528832:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077794.289458:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077794.289465:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077794.289472:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077794.289482:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913529472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077794.289491:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528960:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077794.289492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077794.289500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077794.289502:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077794.289507:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3499 00000100:00080000:3.0:1769077794.289511:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077794.289514:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077794.289518:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077794.289521:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077794.289532:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913529600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077794.289537:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077794/real 1769077794] req@ffff935002a7f800 x1855008913528960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077794.289557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913528960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077794.289565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077794.289567:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077794.289570:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913528960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077794.289575:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077794.289587:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077794.289593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077794/real 1769077794] req@ffff935002a7dc00 x1855008913529088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077794.289601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913529088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077794.289607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077794.289608:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077794.289610:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077794.289614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077794.289624:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077794.289630:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077794/real 1769077794] req@ffff935002a7d180 x1855008913529216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077794.289638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913529216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077794.289643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077794.289645:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077794.289647:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077794.289650:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913529344:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077794.289660:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077794.289666:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077794/real 1769077794] req@ffff935002a7df80 x1855008913529344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077794.289673:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913529344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077794.289679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077794.289680:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077794.289682:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913529344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077794.289696:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913529472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077794.289705:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077794.289711:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077794/real 1769077794] req@ffff935002a7fb80 x1855008913529472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077794.289719:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913529472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077794.289724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077794.289726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077794.289728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913529472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077794.289732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913529600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077794.289740:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077794.289745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077794/real 1769077794] req@ffff935002a7ca80 x1855008913529600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077794.289752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913529600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077794.289774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077794.289776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077794.289778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913529600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077794.481144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077794.993145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077794.993153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077794.993155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077794.993174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077795.313193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077795.505159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077796.017193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077796.017193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077796.017200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077796.017260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077796.337152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077796.529116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077797.041161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077797.041164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077797.041165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077797.041193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077797.361184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077797.553181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077798.065126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077798.065133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077798.065168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077798.065171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077798.385159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077798.577166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077799.089155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077799.089155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077799.089155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077799.089199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077799.409120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077799.409139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077799.409144:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077799.409156:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3505 00000100:00080000:3.0:1769077799.409169:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077799.409177:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077799.409189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769077799.409218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913529728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077799.409248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077799.409252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077799.409254:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077799.409259:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3505 00000100:00080000:3.0:1769077799.409263:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077799.409267:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077799.409278:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077799.409303:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913529728:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077799.409304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913529856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077799.409321:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077799.409325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077799.409328:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077799.409333:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3505 00000100:00080000:3.0:1769077799.409337:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077799.409341:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077799.409348:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077799.409360:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913529984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077799.409372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077799.409378:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077799.409379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077799.409383:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077799.409388:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3505 00000100:00080000:3.0:1769077799.409393:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077799.409398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077799.409404:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077799.409419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913530112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077799.409422:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077799/real 1769077799] req@ffff935002a7e300 x1855008913529728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077799.409451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077799.409452:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913529728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077799.409463:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077799.409467:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077799.409468:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077799.409479:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3505 00000100:00080000:0.0:1769077799.409480:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077799.409485:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913529728:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077799.409489:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077799.409496:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077799.409502:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077799.409507:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077799.409519:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913530240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077799.409528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077799.409539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077799.409541:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077799/real 1769077799] req@ffff935002a7c700 x1855008913529856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077799.409551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077799.409555:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077799.409559:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913529856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077799.409564:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3505 00000100:00080000:0.0:1769077799.409571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077799.409573:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077799.409576:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077799.409583:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529856:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077799.409584:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077799.409589:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529984:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769077799.409593:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077799.409605:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077799.409612:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913530368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077799.409618:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077799/real 1769077799] req@ffff935002a7c380 x1855008913529984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077799.409633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077799.409639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077799.409642:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913529984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077799.409643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077799.409652:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3505 00000100:00080000:3.0:1769077799.409660:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077799.409668:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077799.409689:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077799.409691:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077799.409698:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077799.409701:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913529984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077799.409708:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913530496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077799.409709:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913530112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077799.409734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077799.409745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077799/real 1769077799] req@ffff935002a7f100 x1855008913530112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077799.409754:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913530112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077799.409760:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077799.409762:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077799.409764:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913530112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077799.409778:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913530240:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077799.409789:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077799.409795:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077799/real 1769077799] req@ffff935002a7e300 x1855008913530240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077799.409803:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913530240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077799.409808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077799.409810:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077799.409813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913530240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077799.409817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913530368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077799.409834:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077799.409845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077799/real 1769077799] req@ffff935002a7ca80 x1855008913530368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077799.409858:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913530368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077799.409867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077799.409871:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077799.409874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913530368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077799.409881:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913530496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077799.409898:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077799.409907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077799/real 1769077799] req@ffff935002a7fb80 x1855008913530496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077799.409915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913530496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077799.409921:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077799.409923:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077799.409925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913530496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077799.601166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077800.113127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077800.113130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077800.113136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077800.113185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077800.433177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077800.625114:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077801.137239:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077801.137240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077801.137247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077801.137324:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077801.457149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077801.649154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077802.161204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077802.161210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077802.161253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077802.161261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077802.481180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077802.673243:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077803.185156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077803.185156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077803.185175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077803.185219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077803.505176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077803.697173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077804.209197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077804.209197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077804.209201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077804.209254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077804.529121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077804.529124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077804.529132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077804.529135:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077804.529144:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3510 00000100:00080000:3.0:1769077804.529154:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077804.529160:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077804.529193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913530624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077804.529221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077804.529224:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077804.529225:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077804.529229:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3510 00000100:00080000:3.0:1769077804.529231:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077804.529235:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077804.529242:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077804.529252:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913530624:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077804.529254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913530752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077804.529273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077804.529276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077804.529278:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077804.529281:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3510 00000100:00080000:3.0:1769077804.529284:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077804.529286:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077804.529290:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077804.529298:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913530880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077804.529305:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077804.529312:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077804.529313:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077804.529316:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3510 00000100:00080000:3.0:1769077804.529319:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077804.529324:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077804.529326:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077804.529329:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077804.529337:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913531008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077804.529344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077804.529347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077804.529349:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077804.529352:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3510 00000100:00080000:3.0:1769077804.529354:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077804.529356:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077804.529359:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077804.529369:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913531136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077804.529374:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077804/real 1769077804] req@ffff935002a7df80 x1855008913530624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077804.529380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077804.529388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077804.529391:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077804.529396:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3510 00000100:00080000:3.0:1769077804.529399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077804.529404:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913530624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077804.529404:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077804.529410:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077804.529421:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077804.529422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913531264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077804.529430:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077804.529437:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913530624:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077804.529449:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077804.529452:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077804.529454:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077804.529457:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3510 00000100:00080000:3.0:1769077804.529459:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077804.529462:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077804.529465:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077804.529469:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913530752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077804.529475:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913531392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077804.529504:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077804.529520:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077804/real 1769077804] req@ffff935002a7d180 x1855008913530752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077804.529542:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913530752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077804.529551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077804.529554:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077804.529556:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913530752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077804.529561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913530880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077804.529578:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077804.529611:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077804/real 1769077804] req@ffff935002a7dc00 x1855008913530880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077804.529631:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913530880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077804.529637:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077804.529639:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077804.529641:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913530880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077804.529646:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077804.529660:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077804.529667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077804/real 1769077804] req@ffff935002a7f800 x1855008913531008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077804.529675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913531008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077804.529685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077804.529688:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077804.529690:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077804.529693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913531136:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077804.529703:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077804.529709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077804/real 1769077804] req@ffff935002a7c000 x1855008913531136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077804.529716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913531136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077804.529721:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077804.529723:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077804.529725:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913531136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077804.529728:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913531264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077804.529737:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077804.529743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077804/real 1769077804] req@ffff935002a7fb80 x1855008913531264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077804.529750:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913531264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077804.529755:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077804.529757:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077804.529760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913531264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077804.529774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913531392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077804.529784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077804.529791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077804/real 1769077804] req@ffff935002a7ca80 x1855008913531392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077804.529798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913531392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077804.529803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077804.529805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077804.529807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913531392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077804.721158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077805.233122:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077805.233122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077805.233125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077805.233161:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077805.553163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077805.745159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077806.257161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077806.257161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077806.257200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077806.257362:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077806.577141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077806.769122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077807.281139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077807.281140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077807.281169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077807.281202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077807.601176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077807.793127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077808.305146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077808.305146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077808.305146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077808.305186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077808.625162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077808.817124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077809.329116:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077809.329122:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077809.329122:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077809.329161:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077809.649147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077809.649150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077809.649159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077809.649164:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077809.649180:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3515 00000100:00080000:3.0:1769077809.649193:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077809.649201:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077809.649242:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913531520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077809.649267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077809.649270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077809.649272:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077809.649275:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3515 00000100:00080000:3.0:1769077809.649279:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077809.649283:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077809.649292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077809.649298:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913531520:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077809.649305:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913531648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077809.649314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077809.649317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077809.649318:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077809.649321:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3515 00000100:00080000:3.0:1769077809.649323:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077809.649326:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077809.649330:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077809.649338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913531776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077809.649345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077809.649347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077809.649349:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077809.649352:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3515 00000100:00080000:3.0:1769077809.649354:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077809.649356:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077809.649360:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077809.649363:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077809.649371:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913531904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077809.649385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077809.649388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077809.649389:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077809.649392:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3515 00000100:00080000:3.0:1769077809.649396:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077809.649399:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077809.649402:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077809.649403:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077809/real 1769077809] req@ffff935002a7e300 x1855008913531520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077809.649416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913532032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077809.649436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077809.649438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077809.649442:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913531520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077809.649442:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077809.649450:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3515 00000100:00080000:3.0:1769077809.649455:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077809.649460:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077809.649463:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769077809.649464:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077809.649470:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077809.649476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913532160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077809.649477:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913531520:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077809.649490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077809.649492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077809.649495:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077809.649510:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3515 00000100:00080000:3.0:1769077809.649514:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077809.649517:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077809.649521:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077809.649528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913532288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077809.649537:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531648:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077809.649573:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077809.649584:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077809/real 1769077809] req@ffff935002a7f100 x1855008913531648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077809.649593:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913531648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077809.649600:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077809.649602:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077809.649605:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077809.649609:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077809.649621:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077809.649627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077809/real 1769077809] req@ffff935002a7c380 x1855008913531776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077809.649634:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913531776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077809.649644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077809.649646:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077809.649648:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077809.649651:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077809.649660:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077809.649665:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077809/real 1769077809] req@ffff935002a7c700 x1855008913531904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077809.649672:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913531904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077809.649678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077809.649679:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077809.649681:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913531904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077809.649684:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077809.649693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077809.649699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077809/real 1769077809] req@ffff935002a7ca80 x1855008913532032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077809.649705:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913532032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077809.649711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077809.649713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077809.649716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077809.649719:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077809.649727:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077809.649733:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077809/real 1769077809] req@ffff935002a7fb80 x1855008913532160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077809.649739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913532160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077809.649744:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077809.649746:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077809.649748:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077809.649762:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077809.649771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077809.649777:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077809/real 1769077809] req@ffff935002a7c000 x1855008913532288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077809.649785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913532288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077809.649790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077809.649792:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077809.649793:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077809.841158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077810.353154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077810.353154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077810.353154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077810.353189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077810.673153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077810.865134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077811.377140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077811.377140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077811.377140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077811.377169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077811.697171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077811.889137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077812.401143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077812.401145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077812.401168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077812.401170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077812.721157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077812.913141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077813.425152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077813.425152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077813.425153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077813.425187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077813.745156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077813.937143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077814.449148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077814.449164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077814.449178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077814.449186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077814.769169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077814.769176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077814.769187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077814.769193:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077814.769207:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3520 00000100:00080000:3.0:1769077814.769221:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077814.769231:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077814.769280:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913532416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077814.769319:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077814.769324:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077814.769326:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077814.769331:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3520 00000100:00080000:3.0:1769077814.769335:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077814.769340:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077814.769354:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077814.769370:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913532544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077814.769374:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913532416:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077814.769393:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077814.769397:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077814.769400:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077814.769406:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3520 00000100:00080000:3.0:1769077814.769427:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077814.769434:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077814.769441:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077814.769455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913532672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077814.769467:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077814.769469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077814.769475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077814.769477:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077814.769483:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3520 00000100:00080000:3.0:1769077814.769498:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077814.769503:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077814.769510:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077814.769518:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077814/real 1769077814] req@ffff935002a7f800 x1855008913532416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077814.769527:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913532800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077814.769539:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913532416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077814.769541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077814.769554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077814.769558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077814.769559:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077814.769565:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077814.769569:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3520 00000100:00100000:0.0:1769077814.769572:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913532416:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077814.769576:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077814.769585:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077814.769591:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077814.769603:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913532544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077814.769609:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913532928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077814.769624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077814.769626:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077814.769632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077814.769635:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077814.769640:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077814/real 1769077814] req@ffff935002a7dc00 x1855008913532544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077814.769642:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3520 00000100:00080000:3.0:1769077814.769650:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077814.769654:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077814.769660:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077814.769666:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913532544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077814.769674:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077814.769679:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913533056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077814.769680:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077814.769687:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913532544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077814.769698:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913532672:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077814.769705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077814.769711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077814.769713:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077814.769717:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077814.769720:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3520 00000100:00080000:3.0:1769077814.769728:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077814.769729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077814/real 1769077814] req@ffff935002a7d180 x1855008913532672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077814.769737:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077814.769744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913532672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077814.769746:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077814.769754:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077814.769758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077814.769760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913532672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077814.769767:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913532800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077814.769767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913533184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077814.769794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077814.769806:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077814/real 1769077814] req@ffff935002a7df80 x1855008913532800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077814.769822:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913532800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077814.769831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077814.769835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077814.769839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913532800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077814.769873:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532928:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077814.769895:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077814.769909:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077814/real 1769077814] req@ffff935002a7f800 x1855008913532928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077814.769920:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913532928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077814.769926:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077814.769928:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077814.769931:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913532928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077814.769935:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077814.769949:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077814.769956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077814/real 1769077814] req@ffff935002a7c000 x1855008913533056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077814.769964:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913533056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077814.769969:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077814.769971:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077814.769973:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077814.769977:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077814.769985:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077814.769991:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077814/real 1769077814] req@ffff935002a7fb80 x1855008913533184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077814.769998:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913533184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077814.770004:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077814.770007:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077814.770009:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077814.961229:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077815.473178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077815.473178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077815.473180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077815.473236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077815.793180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077815.985165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077816.497158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077816.497159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077816.497159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077816.497224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077816.817184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077817.009291:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077817.521172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077817.521172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077817.521200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077817.521253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077817.841188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077818.033163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077818.545199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077818.545199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077818.545208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077818.545243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077818.865124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077819.057212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077819.569150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077819.569150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077819.569150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077819.569325:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077819.889166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077819.889170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077819.889179:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077819.889185:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077819.889199:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3525 00000100:00080000:3.0:1769077819.889212:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077819.889221:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077819.889272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913533312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077819.889298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077819.889302:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077819.889304:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077819.889307:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3525 00000100:00080000:3.0:1769077819.889311:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077819.889315:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077819.889338:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077819.889344:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913533312:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077819.889352:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913533440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077819.889367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077819.889370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077819.889371:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077819.889380:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3525 00000100:00080000:3.0:1769077819.889383:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077819.889386:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077819.889391:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077819.889399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913533568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077819.889406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077819.889408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077819.889409:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077819.889412:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3525 00000100:00080000:3.0:1769077819.889414:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077819.889420:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077819.889420:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077819.889426:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077819.889435:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913533696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077819.889442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077819.889445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077819.889446:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077819.889449:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3525 00000100:00080000:3.0:1769077819.889452:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077819.889454:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077819.889457:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077819.889466:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913533824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077819.889473:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff935002a7ca80 x1855008913533312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077819.889475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077819.889483:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077819.889486:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077819.889493:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3525 00000100:00080000:3.0:1769077819.889496:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077819.889499:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077819.889503:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077819.889504:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913533312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077819.889514:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913533952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077819.889515:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077819.889524:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077819.889527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077819.889531:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913533312:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077819.889532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077819.889537:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077819.889541:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3525 00000100:00080000:3.0:1769077819.889543:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077819.889546:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077819.889548:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077819.889556:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913534080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077819.889559:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913533440:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077819.889584:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077819.889596:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff935002a7c700 x1855008913533440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077819.889605:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913533440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077819.889620:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077819.889622:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077819.889628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913533440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077819.889632:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913533568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077819.889645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077819.889652:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff935002a7c380 x1855008913533568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077819.889660:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913533568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077819.889666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077819.889668:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077819.889670:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913533568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077819.889674:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913533696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077819.889683:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077819.889689:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff935002a7f100 x1855008913533696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077819.889696:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913533696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077819.889702:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077819.889704:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077819.889706:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913533696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077819.889709:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077819.889718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077819.889723:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff935002a7e300 x1855008913533824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077819.889730:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913533824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077819.889735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077819.889737:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077819.889739:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077819.889742:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077819.889751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077819.889756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff935002a7fb80 x1855008913533952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077819.889763:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913533952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077819.889768:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077819.889770:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077819.889772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913533952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077819.889789:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077819.889799:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077819.889805:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff935002a7c000 x1855008913534080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077819.889812:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913534080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077819.889818:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077819.889819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077819.889821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077820.081129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077820.593146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077820.593146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077820.593147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077820.593183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077820.913203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077821.105170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077821.617134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077821.617135:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077821.617135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077821.617172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077821.937176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077822.129177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077822.641158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077822.641158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077822.641158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077822.641194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077822.961206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077823.153233:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077823.665186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077823.665186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077823.665187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077823.665224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077823.985180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077824.177203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077824.689144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077824.689144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077824.689147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077824.689177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077825.009187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077825.009191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077825.009202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077825.009208:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077825.009221:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3530 00000100:00080000:3.0:1769077825.009234:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077825.009255:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077825.009302:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913534208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077825.009333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077825.009337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077825.009339:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077825.009344:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3530 00000100:00080000:3.0:1769077825.009348:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077825.009354:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077825.009366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077825.009375:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913534208:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077825.009382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913534336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077825.009395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077825.009406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077825.009409:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077825.009413:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3530 00000100:00080000:3.0:1769077825.009418:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077825.009422:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077825.009429:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077825.009441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913534464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077825.009452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077825.009456:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769077825.009459:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077825.009460:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077825.009469:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3530 00000100:00080000:3.0:1769077825.009473:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077825.009478:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077825.009483:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077825.009495:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913534592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077825.009506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077825.009510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077825.009512:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077825.009518:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077825/real 1769077825] req@ffff935002a7f800 x1855008913534208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077825.009521:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3530 00000100:00080000:3.0:1769077825.009526:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077825.009534:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077825.009549:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077825.009557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913534208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077825.009567:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913534720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077825.009573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077825.009581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077825.009583:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077825.009589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769077825.009593:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913534208:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077825.009594:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077825.009603:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3530 00000100:00080000:3.0:1769077825.009608:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077825.009612:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077825.009617:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077825.009629:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913534848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077825.009632:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913534336:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077825.009650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077825.009654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077825.009657:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077825.009662:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3530 00000100:00080000:3.0:1769077825.009666:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077825.009667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077825.009673:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077825.009678:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077825.009685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077825/real 1769077825] req@ffff935002a7df80 x1855008913534336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077825.009693:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913534976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077825.009714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913534336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077825.009726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077825.009731:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077825.009736:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913534336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077825.009745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913534464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077825.009764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077825.009773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077825/real 1769077825] req@ffff935002a7d180 x1855008913534464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077825.009786:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913534464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077825.009801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077825.009805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077825.009809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913534464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077825.009815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913534592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077825.009830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077825.009839:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077825/real 1769077825] req@ffff935002a7dc00 x1855008913534592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077825.009851:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913534592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077825.009860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077825.009863:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077825.009867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913534592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077825.009873:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534720:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077825.009888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077825.009897:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077825/real 1769077825] req@ffff935002a7c000 x1855008913534720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077825.009909:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913534720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077825.009918:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077825.009921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077825.009925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077825.009944:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077825.009960:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077825.009969:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077825/real 1769077825] req@ffff935002a7fb80 x1855008913534848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077825.009981:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913534848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077825.009989:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077825.009992:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077825.009996:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077825.010001:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077825.010015:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077825.010024:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077825/real 1769077825] req@ffff935002a7e300 x1855008913534976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077825.010036:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913534976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077825.010045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077825.010066:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077825.010071:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913534976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077825.201257:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077825.713185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077825.713185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077825.713188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077825.713271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077826.033151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077826.225126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077826.737173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077826.737174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077826.737174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077826.737225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077827.057165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077827.249188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077827.761172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077827.761172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077827.761178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077827.761254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077828.081157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077828.273126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077828.785165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077828.785165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077828.785206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077828.785270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077829.105165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077829.297140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077829.809186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077829.809186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077829.809186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077829.809247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077830.129216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077830.129229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077830.129237:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077830.129251:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077830.129261:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3535 00000100:00080000:3.0:1769077830.129273:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077830.129280:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077830.129321:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913535104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077830.129350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077830.129353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077830.129354:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077830.129358:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3535 00000100:00080000:3.0:1769077830.129362:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077830.129366:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077830.129375:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077830.129380:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913535104:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077830.129398:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913535232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077830.129408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077830.129412:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077830.129413:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077830.129416:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3535 00000100:00080000:3.0:1769077830.129420:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077830.129422:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077830.129426:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077830.129434:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913535360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077830.129441:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077830.129446:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077830.129447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077830.129451:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077830.129454:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3535 00000100:00080000:3.0:1769077830.129458:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077830.129461:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077830.129464:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077830.129474:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913535488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077830.129481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077830.129484:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077830.129486:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077830.129490:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077830/real 1769077830] req@ffff935002a7f100 x1855008913535104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077830.129492:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3535 00000100:00080000:3.0:1769077830.129497:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077830.129504:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077830.129510:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077830.129521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913535616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077830.129528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077830.129534:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913535104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077830.129538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077830.129542:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077830.129550:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3535 00000100:00080000:0.0:1769077830.129554:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077830.129554:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077830.129558:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077830.129563:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769077830.129564:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077830.129568:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913535104:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077830.129577:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913535744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077830.129586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077830.129588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077830.129590:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077830.129592:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3535 00000100:00080000:3.0:1769077830.129595:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077830.129596:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913535232:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077830.129601:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077830.129606:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077830.129616:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913535872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077830.129626:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077830.129639:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077830/real 1769077830] req@ffff935002a7c380 x1855008913535232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077830.129649:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913535232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077830.129655:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077830.129657:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077830.129660:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913535232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077830.129664:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913535360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077830.129677:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077830.129683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077830/real 1769077830] req@ffff935002a7c700 x1855008913535360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077830.129691:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913535360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077830.129697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077830.129699:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077830.129701:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913535360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077830.129704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913535488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077830.129714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077830.129719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077830/real 1769077830] req@ffff935002a7ca80 x1855008913535488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077830.129727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913535488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077830.129733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077830.129735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077830.129738:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913535488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077830.129744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913535616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077830.129763:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077830.129775:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077830/real 1769077830] req@ffff935002a7e300 x1855008913535616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077830.129789:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913535616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077830.129799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077830.129802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077830.129805:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913535616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077830.129809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913535744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077830.129825:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077830.129833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077830/real 1769077830] req@ffff935002a7fb80 x1855008913535744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077830.129850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913535744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077830.129856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077830.129859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077830.129861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913535744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077830.129876:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913535872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077830.129888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077830.129895:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077830/real 1769077830] req@ffff935002a7c000 x1855008913535872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077830.129902:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913535872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077830.129908:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077830.129917:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077830.129919:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913535872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077830.321123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077830.833214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077830.833214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077830.833221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077830.833283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077831.153219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077831.345205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077831.857165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077831.857165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077831.857178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077831.857276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077832.177184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077832.369146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077832.881165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077832.881165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077832.881171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077832.881232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077833.201176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077833.393187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077833.905186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077833.905187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077833.905247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077833.905298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077834.225191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077834.417219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077834.929162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077834.929162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077834.929171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077834.929224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077835.249195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077835.249240:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077835.249247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077835.249251:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077835.249262:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3540 00000100:00080000:1.0:1769077835.249275:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077835.249283:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077835.249329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913536000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077835.249363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077835.249368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077835.249370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077835.249375:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3540 00000100:00080000:1.0:1769077835.249379:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077835.249383:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077835.249397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077835.249410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913536128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077835.249424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077835.249428:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913536000:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077835.249429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077835.249432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077835.249442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3540 00000100:00080000:1.0:1769077835.249446:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077835.249449:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077835.249454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077835.249466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913536256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077835.249475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077835.249480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077835.249482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077835.249486:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3540 00000100:00080000:1.0:1769077835.249504:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077835.249508:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077835.249514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077835.249521:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077835.249527:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913536384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077835.249540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077835.249544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077835.249547:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077835.249552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3540 00000100:00080000:1.0:1769077835.249555:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077835.249559:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077835.249566:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077835.249576:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077835/real 1769077835] req@ffff9350100ca680 x1855008913536000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077835.249579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913536512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077835.249609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077835.249613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077835.249615:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077835.249619:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913536000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077835.249622:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3540 00000100:00080000:1.0:1769077835.249627:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077835.249634:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077835.249641:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077835.249643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077835.249650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769077835.249656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913536640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077835.249658:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913536000:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077835.249673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077835.249677:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077835.249680:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077835.249684:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3540 00000100:00080000:1.0:1769077835.249688:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077835.249692:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913536128:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077835.249694:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077835.249703:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077835.249715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913536768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077835.249727:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077835.249744:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077835/real 1769077835] req@ffff9350100c8e00 x1855008913536128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077835.249759:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913536128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077835.249770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077835.249774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077835.249777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913536128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077835.249784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913536256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077835.249799:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077835.249808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077835/real 1769077835] req@ffff9350100c9880 x1855008913536256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077835.249820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913536256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077835.249829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077835.249832:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077835.249836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913536256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077835.249841:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913536384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077835.249855:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077835.249863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077835/real 1769077835] req@ffff9350100c9500 x1855008913536384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077835.249874:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913536384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077835.249882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077835.249885:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077835.249889:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913536384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077835.249894:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913536512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077835.249907:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077835.249916:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077835/real 1769077835] req@ffff9350100caa00 x1855008913536512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077835.249927:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913536512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077835.249943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077835.249947:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077835.249950:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913536512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077835.249956:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913536640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077835.249972:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077835.249981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077835/real 1769077835] req@ffff9350100c8380 x1855008913536640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077835.249993:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913536640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077835.250002:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077835.250005:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077835.250009:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913536640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077835.250027:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913536768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077835.250040:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077835.250086:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077835/real 1769077835] req@ffff9350100ca300 x1855008913536768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077835.250099:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913536768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077835.250108:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077835.250111:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077835.250115:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913536768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077835.441136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077835.953190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077835.953190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077835.953190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077835.953230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077836.273163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077836.465205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077836.977158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077836.977176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077836.977177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077836.977235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077837.297196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077837.489173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077838.001207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077838.001207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077838.001327:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077838.001364:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077838.321127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077838.513148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077839.025178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077839.025231:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077839.025259:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077839.025269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077839.345174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077839.537165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077840.049207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077840.049240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077840.049264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077840.049271:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:0.0:1769077840.369247:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077840.369261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077840.369267:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077840.369278:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3546 00000100:00080000:0.0:1769077840.369292:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077840.369298:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077840.369300:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077840.369344:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913536896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077840.369378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077840.369384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077840.369386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077840.369392:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3546 00000100:00080000:0.0:1769077840.369397:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077840.369402:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077840.369414:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077840.369433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913537024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077840.369445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077840.369449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077840.369452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077840.369460:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3546 00000100:00100000:2.0:1769077840.369468:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913536896:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077840.369469:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077840.369484:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077840.369492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077840.369511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913537152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077840.369525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077840.369530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077840.369532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077840.369535:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3546 00000100:00080000:0.0:1769077840.369540:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077840.369546:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077840.369551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077840.369561:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077840.369569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913537280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077840.369582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077840.369587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077840.369590:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077840.369595:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3546 00000100:00080000:0.0:1769077840.369599:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077840.369605:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077840.369614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077840.369617:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077840/real 1769077840] req@ffff9350103fb100 x1855008913536896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077840.369635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913536896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077840.369642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913537408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077840.369647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077840.369658:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077840.369670:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913536896:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077840.369676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077840.369681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077840.369690:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077840.369698:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537024:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077840.369706:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3546 00000100:00080000:0.0:1769077840.369715:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077840.369721:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077840.369726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:0.0:1769077840.369729:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077840.369739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077840/real 1769077840] req@ffff9350103faa00 x1855008913537024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077840.369757:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913537024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077840.369759:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913537536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077840.369798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077840.369808:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769077840.369813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077840.369814:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537024:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077840.369828:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077840.369832:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077840.369841:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3546 00000100:00080000:0.0:1769077840.369848:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077840.369855:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537152:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077840.369857:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077840.369871:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077840.369887:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077840.369895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913537664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077840.369907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077840/real 1769077840] req@ffff9350103fb480 x1855008913537152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077840.369932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913537152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077840.369939:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077840.369946:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077840.369951:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077840.369959:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077840.370011:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077840.370021:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077840/real 1769077840] req@ffff9350103fad80 x1855008913537280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077840.370030:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913537280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077840.370035:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077840.370037:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077840.370040:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077840.370103:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913537408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077840.370118:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077840.370126:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077840/real 1769077840] req@ffff9350103f8700 x1855008913537408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077840.370135:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913537408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077840.370142:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077840.370146:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077840.370149:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913537408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077840.370156:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913537536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077840.370176:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077840.370188:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077840/real 1769077840] req@ffff9350103fb800 x1855008913537536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077840.370200:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913537536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077840.370210:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077840.370212:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077840.370214:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913537536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077840.370243:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913537664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077840.370265:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077840.370277:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077840/real 1769077840] req@ffff9350103faa00 x1855008913537664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077840.370290:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913537664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077840.370299:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077840.370303:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077840.370307:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913537664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077840.561179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077841.073124:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077841.073166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077841.073236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077841.073311:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077841.393135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077841.585165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077842.097195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077842.097201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077842.097235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077842.097264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077842.417181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077842.609170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077843.121162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077843.121162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077843.121162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077843.121269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077843.441135:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077843.633139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077844.145152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077844.145152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077844.145167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077844.145216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077844.465121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077844.657191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077845.169213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077845.169213:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077845.169243:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077845.169278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077845.489150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077845.489158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077845.489164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077845.489171:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077845.489183:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3551 00000100:00080000:0.0:1769077845.489195:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077845.489204:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077845.489251:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913537792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077845.489289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077845.489293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077845.489295:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077845.489300:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3551 00000100:00080000:0.0:1769077845.489304:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077845.489309:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077845.489321:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077845.489338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913537920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077845.489349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077845.489352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077845.489355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077845.489360:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3551 00000100:00080000:0.0:1769077845.489364:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077845.489368:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077845.489375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077845.489376:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913537792:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077845.489391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913538048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077845.489408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077845.489411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077845.489413:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077845.489417:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3551 00000100:00080000:0.0:1769077845.489420:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077845.489423:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077845.489426:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077845.489435:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913538176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077845.489442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077845.489444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077845.489446:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077845.489451:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077845.489454:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3551 00000100:00080000:0.0:1769077845.489459:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077845.489462:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077845.489466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077845.489475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913538304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077845.489482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077845.489485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077845.489486:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077845.489489:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3551 00000100:00080000:0.0:1769077845.489492:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077845.489494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077845.489497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077845.489499:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077845/real 1769077845] req@ffff9350103fb100 x1855008913537792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077845.489507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913538432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077845.489525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077845.489528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077845.489530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077845.489533:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3551 00000100:00080000:0.0:1769077845.489535:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077845.489539:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913537792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077845.489540:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077845.489546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077845.489557:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913538560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077845.489560:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077845.489571:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077845.489576:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913537792:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077845.489583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537920:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077845.489605:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077845.489618:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077845/real 1769077845] req@ffff9350103f8380 x1855008913537920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077845.489632:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913537920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077845.489648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077845.489652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077845.489655:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913537920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077845.489684:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077845.489701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077845.489709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077845/real 1769077845] req@ffff9350103f9180 x1855008913538048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077845.489716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913538048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077845.489722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077845.489724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077845.489726:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077845.489729:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077845.489738:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077845.489744:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077845/real 1769077845] req@ffff9350103f9500 x1855008913538176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077845.489750:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913538176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077845.489756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077845.489758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077845.489760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077845.489764:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913538304:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077845.489773:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077845.489778:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077845/real 1769077845] req@ffff9350103fbb80 x1855008913538304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077845.489785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913538304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077845.489790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077845.489792:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077845.489794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913538304:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077845.489797:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913538432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077845.489806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077845.489811:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077845/real 1769077845] req@ffff9350103f9c00 x1855008913538432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077845.489818:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913538432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077845.489823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077845.489825:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077845.489827:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913538432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077845.489830:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913538560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077845.489838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077845.489844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077845/real 1769077845] req@ffff9350103f9f80 x1855008913538560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077845.489864:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913538560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077845.489870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077845.489872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077845.489874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913538560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077845.681134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077846.193124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077846.193126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077846.193126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077846.193170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077846.513149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077846.705150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077847.217202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077847.217202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077847.217221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077847.217267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077847.537121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077847.729124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077848.241166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077848.241166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077848.241166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077848.241206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077848.561150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077848.753173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077849.265125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077849.265149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077849.265151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077849.265228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077849.585156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077849.777123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077850.289142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077850.289142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077850.289143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077850.289217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077850.609153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077850.609161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077850.609168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077850.609171:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077850.609179:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3556 00000100:00080000:3.0:1769077850.609201:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077850.609208:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077850.609241:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913538688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077850.609278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077850.609281:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077850.609283:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077850.609286:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3556 00000100:00080000:3.0:1769077850.609288:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077850.609292:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077850.609301:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077850.609311:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913538816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077850.609319:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077850.609322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077850.609323:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077850.609327:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3556 00000100:00100000:2.0:1769077850.609331:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913538688:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077850.609331:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077850.609337:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077850.609343:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077850.609352:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913538944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077850.609359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077850.609362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077850.609363:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077850.609366:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3556 00000100:00080000:3.0:1769077850.609368:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077850.609371:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077850.609374:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077850.609381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913539072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077850.609388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077850.609391:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077850.609392:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077850.609395:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3556 00000100:00080000:3.0:1769077850.609399:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077850.609403:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077850.609408:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077850.609422:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913539200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077850.609442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077850.609443:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538816:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077850.609452:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077850.609455:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077850.609461:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3556 00000100:00080000:3.0:1769077850.609473:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077850.609477:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077850.609483:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077850.609486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077850.609514:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913539328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077850.609520:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539072:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077850.609537:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077850.609542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077850.609544:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913539200:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077850.609546:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077850.609564:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3556 00000100:00080000:3.0:1769077850.609571:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077850.609575:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077850.609580:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077850.609591:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913539328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769077850.609593:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913539456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077850.609621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913539456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077850.801133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077851.313166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077851.313166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077851.313166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077851.313206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077851.633126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077851.825119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077852.337162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077852.337162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077852.337162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077852.337200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077852.657123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077852.849139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077853.361150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077853.361150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077853.361159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077853.361204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077853.681165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:0.0:1769077853.681322:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:0.0:1769077853.681358:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077853.681374:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769077853.681405:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769077853.681411:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769077853.681417:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769077853.681426:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769077853.681436:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769077853.681447:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077853.681458:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077850/real 1769077853] req@ffff935002a7fb80 x1855008913539328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077853.681485:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913539328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077853.681505:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077853.681511:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077853.681516:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913539328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077853.681525:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077850/real 1769077853] req@ffff935002a7df80 x1855008913538944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077853.681538:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913538944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077853.681548:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077853.681552:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077853.681555:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077853.681564:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077850/real 1769077853] req@ffff935002a7f800 x1855008913539072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077853.681577:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913539072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077853.681586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077853.681590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077853.681593:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077853.681600:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077850/real 1769077853] req@ffff935002a7c000 x1855008913539200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077853.681613:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913539200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077853.681622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077853.681626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077853.681629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913539200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077853.681637:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077850/real 1769077853] req@ffff935002a7dc00 x1855008913538688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077853.681652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913538688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077853.681661:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077853.681665:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077853.681668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913538688:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077853.681675:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077850/real 1769077853] req@ffff935002a7d180 x1855008913538816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077853.681687:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913538816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077853.681697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077853.681700:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077853.681703:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913538816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077853.681740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077850/real 1769077853] req@ffff935002a7e300 x1855008913539456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077853.681753:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913539456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077853.681779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077853.681784:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077853.681788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913539456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077853.873166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077854.385127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077854.385127:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077854.385228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077854.385245:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077854.705131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077854.897162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077855.409134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077855.409156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077855.409169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077855.409192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769077855.729119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077855.729120:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077855.729128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077855.729132:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077855.729142:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3561 00000100:00080000:3.0:1769077855.729153:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077855.729161:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077855.729198:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913539584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077855.729227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077855.729231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077855.729234:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077855.729238:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3561 00000100:00080000:3.0:1769077855.729242:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077855.729246:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077855.729257:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077855.729261:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913539584:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077855.729277:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913539712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077855.729293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077855.729296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077855.729298:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077855.729302:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3561 00000100:00080000:3.0:1769077855.729314:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077855.729317:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077855.729323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769077855.729324:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077855.729334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913539840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077855.729343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077855.729346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077855.729348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077855.729351:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3561 00000100:00080000:3.0:1769077855.729354:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077855.729358:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077855.729360:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077855/real 1769077855] req@ffff935002a7ca80 x1855008913539584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077855.729363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077855.729372:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913539968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077855.729377:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913539584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077855.729382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077855.729386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077855.729388:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077855.729389:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077855.729392:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3561 00000100:00080000:2.0:1769077855.729393:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077855.729395:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077855.729398:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913539584:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077855.729400:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077855.729404:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077855.729421:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077855.729421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913540096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077855.729431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077855.729435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077855.729437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077855.729440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3561 00000800:00000100:2.0:1769077855.729443:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077855.729444:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077855.729448:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077855.729453:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077855.729455:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077855/real 1769077855] req@ffff935002a7c700 x1855008913539712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077855.729464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913540224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077855.729468:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913539712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077855.729473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077855.729477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077855.729479:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077855.729479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077855.729483:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077855.729484:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3561 00000100:00100000:2.0:1769077855.729487:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539712:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077855.729487:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077855.729494:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539840:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077855.729506:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077855.729512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077855.729523:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913540352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077855.729529:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077855.729540:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077855/real 1769077855] req@ffff935002a7c380 x1855008913539840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077855.729552:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913539840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077855.729562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077855.729566:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077855.729570:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077855.729577:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077855.729616:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077855.729630:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077855/real 1769077855] req@ffff935002a7f100 x1855008913539968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077855.729643:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913539968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077855.729651:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077855.729654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077855.729658:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913539968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077855.729691:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540096:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077855.729714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077855.729741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077855/real 1769077855] req@ffff935002a7e300 x1855008913540096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077855.729753:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913540096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077855.729762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077855.729765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077855.729771:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077855.729781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077855.729805:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077855.729816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077855/real 1769077855] req@ffff935002a7d180 x1855008913540224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077855.729829:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913540224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077855.729837:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077855.729841:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077855.729845:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077855.729851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077855.729871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077855.729884:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077855/real 1769077855] req@ffff935002a7dc00 x1855008913540352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077855.729898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913540352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077855.729906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077855.729909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077855.729913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077855.921132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077856.433131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077856.433138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077856.433161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077856.433178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077856.753218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077856.945139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077857.457140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077857.457140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077857.457182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077857.457186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077857.777178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077857.969164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077858.481163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077858.481163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077858.481171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077858.481200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077858.801146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077858.993120:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077859.505168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077859.505168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077859.505169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077859.505206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077859.825163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077860.017145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077860.529124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077860.529124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077860.529129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077860.529167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077860.849151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077860.849151:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077860.849164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077860.849171:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077860.849183:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3566 00000100:00080000:3.0:1769077860.849196:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077860.849204:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077860.849254:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913540480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077860.849289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077860.849293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077860.849295:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077860.849300:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3566 00000100:00080000:3.0:1769077860.849304:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077860.849308:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077860.849320:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077860.849334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913540608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077860.849346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077860.849350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077860.849353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077860.849359:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3566 00000100:00080000:3.0:1769077860.849362:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077860.849368:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077860.849374:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913540480:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077860.849378:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077860.849400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913540736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077860.849414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077860.849419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077860.849421:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077860.849426:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3566 00000100:00080000:3.0:1769077860.849446:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077860.849454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077860.849455:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077860.849467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077860.849484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913540864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077860.849497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077860.849501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077860.849503:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077860.849509:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077860/real 1769077860] req@ffff935002a7c000 x1855008913540480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077860.849514:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3566 00000100:00080000:3.0:1769077860.849522:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077860.849535:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077860.849544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077860.849553:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913540480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077860.849564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913540992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077860.849567:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.849580:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077860.849595:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913540480:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077860.849595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077860.849614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077860.849618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913540608:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077860.849620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077860.849630:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3566 00000100:00080000:3.0:1769077860.849651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077860.849658:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077860.849665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077860.849665:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077860.849681:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077860/real 1769077860] req@ffff935002a7f800 x1855008913540608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077860.849691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913541120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077860.849698:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913540608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.849735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.849741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077860.849742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077860.849748:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913540608:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077860.849754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077860.849759:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077860.849767:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3566 00000100:00080000:3.0:1769077860.849773:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077860.849778:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077860.849783:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913540736:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769077860.849790:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077860.849803:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077860.849813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913541248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077860.849821:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077860/real 1769077860] req@ffff935002a7df80 x1855008913540736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.849855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913540736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.849865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.849869:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077860.849873:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913540736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077860.849877:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913540864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077860.849895:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077860.849906:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077860/real 1769077860] req@ffff935002a7fb80 x1855008913540864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.849919:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913540864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.849928:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.849931:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077860.849935:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913540864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077860.849942:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540992:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077860.849959:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077860.849967:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077860/real 1769077860] req@ffff935002a7dc00 x1855008913540992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.849975:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913540992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.849980:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.849982:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077860.849985:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913540992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077860.849988:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913541120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077860.849998:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077860.850003:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077860/real 1769077860] req@ffff935002a7d180 x1855008913541120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.850011:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913541120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.850016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.850017:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077860.850019:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913541120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077860.850032:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913541248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077860.850042:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077860.850083:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077860/real 1769077860] req@ffff935002a7e300 x1855008913541248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.850092:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913541248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.850098:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.850100:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077860.850102:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913541248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077861.041203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077861.553159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077861.553166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077861.553174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077861.553212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077861.873210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077862.065206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077862.577191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077862.577191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077862.577218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077862.577230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077862.897184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077863.089150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077863.601168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077863.601181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077863.601207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077863.601217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077863.921161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077864.113179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077864.625215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077864.625232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077864.625242:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077864.625265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077864.945162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077865.137124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077865.649159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077865.649180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077865.649188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077865.649195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077865.969196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077865.969196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077865.969210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077865.969215:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077865.969228:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3571 00000100:00080000:2.0:1769077865.969241:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077865.969249:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077865.969310:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913541376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077865.969345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077865.969350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077865.969353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077865.969358:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3571 00000100:00080000:2.0:1769077865.969362:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077865.969367:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077865.969382:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077865.969397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913541504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077865.969404:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913541376:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077865.969418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077865.969423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077865.969425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077865.969432:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3571 00000100:00080000:2.0:1769077865.969436:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077865.969441:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077865.969447:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077865.969461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913541632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077865.969472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077865.969476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077865.969479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077865.969483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3571 00000100:00080000:2.0:1769077865.969488:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077865.969493:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077865.969497:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077865.969505:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077865.969518:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913541760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077865.969529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077865.969533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077865.969537:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077865.969542:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3571 00000100:00080000:2.0:1769077865.969547:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077865.969570:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff935002a62300 x1855008913541376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077865.969582:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077865.969591:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077865.969599:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913541376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077865.969609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913541888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077865.969615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077865.969626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077865.969626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077865.969634:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913541376:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077865.969634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077865.969639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077865.969647:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3571 00000100:00080000:2.0:1769077865.969652:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077865.969656:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077865.969662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077865.969670:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913541504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077865.969677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913542016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077865.969692:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077865.969695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077865.969698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077865.969706:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077865.969706:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3571 00000100:00080000:2.0:1769077865.969724:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077865.969727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff935002a61880 x1855008913541504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077865.969732:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077865.969744:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077865.969752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913541504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077865.969761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913542144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077865.969766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077865.969777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077865.969783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913541504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077865.969791:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913541632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077865.969815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077865.969828:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff935002a61180 x1855008913541632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077865.969841:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913541632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077865.969850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077865.969854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077865.969857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913541632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077865.969863:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913541760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077865.969884:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077865.969896:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff935002a62680 x1855008913541760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077865.969908:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913541760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077865.969917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077865.969921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077865.969924:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913541760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077865.969931:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913541888:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077865.969949:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077865.969962:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff935002a60a80 x1855008913541888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077865.969975:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913541888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077865.969983:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077865.969986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077865.969990:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913541888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077865.970010:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077865.970029:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077865.970040:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff935002a61f80 x1855008913542016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077865.970088:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913542016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077865.970097:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077865.970101:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077865.970105:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077865.970111:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077865.970133:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077865.970145:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff935002a61c00 x1855008913542144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077865.970165:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913542144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077865.970174:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077865.970177:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077865.970181:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077866.161168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077866.673223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077866.673223:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077866.673224:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077866.673252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077866.993207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077867.185153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077867.697159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077867.697159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077867.697159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077867.697259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077868.017201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077868.209176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077868.721183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077868.721183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077868.721183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077868.721412:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077869.041193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077869.233117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077869.745170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077869.745170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077869.745171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077869.745221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077870.065245:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077870.257187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077870.769177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077870.769177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077870.769284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077870.769307:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077871.089152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077871.089163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077871.089171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077871.089175:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077871.089185:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3576 00000100:00080000:3.0:1769077871.089196:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077871.089203:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077871.089245:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913542272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077871.089271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077871.089274:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077871.089275:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077871.089278:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3576 00000100:00080000:3.0:1769077871.089280:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077871.089283:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077871.089292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077871.089301:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913542400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077871.089309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077871.089312:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077871.089314:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077871.089317:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3576 00000100:00080000:3.0:1769077871.089319:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077871.089324:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913542272:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077871.089324:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077871.089332:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077871.089339:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913542528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077871.089353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077871.089356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077871.089357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077871.089361:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3576 00000100:00080000:3.0:1769077871.089363:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077871.089365:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077871.089369:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077871.089377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913542656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077871.089383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077871.089386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077871.089387:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077871.089390:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3576 00000100:00080000:3.0:1769077871.089392:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077871.089394:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077871.089397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077871.089400:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077871.089410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913542784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077871.089417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077871.089420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077871.089421:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077871.089424:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3576 00000100:00080000:3.0:1769077871.089433:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077871.089436:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077871.089439:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077871.089447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913542912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077871.089453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077871.089455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077871.089457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077871.089462:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3576 00000100:00100000:0.0:1769077871.089464:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077871/real 1769077871] req@ffff935002a7f100 x1855008913542272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077871.089464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077871.089494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077871.089499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077871.089504:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913542272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077871.089511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913543040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077871.089525:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077871.089533:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077871.089539:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913542272:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077871.089548:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913542400:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077871.089573:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077871.089585:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077871/real 1769077871] req@ffff935002a7c380 x1855008913542400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077871.089594:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913542400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077871.089601:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077871.089603:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077871.089605:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913542400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077871.089630:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913542528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077871.089642:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077871.089649:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077871/real 1769077871] req@ffff935002a7c700 x1855008913542528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077871.089657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913542528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077871.089662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077871.089663:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077871.089665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913542528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077871.089669:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913542656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077871.089678:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077871.089683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077871/real 1769077871] req@ffff935002a7ca80 x1855008913542656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077871.089690:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913542656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077871.089696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077871.089698:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077871.089700:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913542656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077871.089704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542784:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077871.089713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077871.089718:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077871/real 1769077871] req@ffff935002a7e300 x1855008913542784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077871.089725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913542784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077871.089730:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077871.089732:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077871.089733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077871.089736:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077871.089746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077871.089751:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077871/real 1769077871] req@ffff935002a7d180 x1855008913542912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077871.089759:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913542912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077871.089763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077871.089765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077871.089767:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913542912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077871.089771:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077871.089781:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077871.089787:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077871/real 1769077871] req@ffff935002a7dc00 x1855008913543040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077871.089794:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913543040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077871.089799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077871.089801:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077871.089803:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077871.281176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077871.793201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077871.793226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077871.793239:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077871.793241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077872.113164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077872.305139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077872.817128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077872.817128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077872.817128:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077872.817183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077873.137118:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077873.329304:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077873.841150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077873.841150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077873.841194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077873.841313:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077874.161176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077874.353128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077874.865156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077874.865156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077874.865157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077874.865189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077875.185138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077875.377191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077875.889123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077875.889125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077875.889131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077875.889168:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077876.209162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077876.209162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077876.209174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077876.209179:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077876.209190:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3581 00000100:00080000:3.0:1769077876.209209:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077876.209217:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077876.209250:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913543168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077876.209277:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077876.209280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077876.209282:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077876.209285:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3581 00000100:00080000:3.0:1769077876.209288:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077876.209291:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077876.209298:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077876.209307:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913543296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077876.209315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077876.209318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077876.209319:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077876.209322:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3581 00000100:00080000:3.0:1769077876.209324:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077876.209326:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077876.209331:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077876.209337:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913543168:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769077876.209342:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913543424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077876.209353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077876.209355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077876.209356:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077876.209359:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3581 00000100:00080000:3.0:1769077876.209367:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077876.209369:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077876.209373:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077876.209398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913543552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077876.209406:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077876.209409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077876.209411:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077876.209414:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3581 00000100:00080000:3.0:1769077876.209417:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077876.209421:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077876.209426:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077876.209428:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077876.209443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913543680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077876.209450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077876.209452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077876.209454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077876.209458:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3581 00000100:00080000:3.0:1769077876.209461:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077876.209464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077876/real 1769077876] req@ffff935002a7fb80 x1855008913543168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077876.209475:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077876.209486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913543808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077876.209496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077876.209498:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913543168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077876.209501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077876.209507:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077876.209517:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3581 00000100:00080000:0.0:1769077876.209522:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077876.209524:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077876.209534:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077876.209535:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209545:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913543168:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769077876.209546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077876.209557:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913543296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077876.209566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913543936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077876.209583:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209595:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077876/real 1769077876] req@ffff935002a7df80 x1855008913543296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077876.209605:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913543296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077876.209612:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077876.209614:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077876.209617:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913543296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077876.209646:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913543424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077876.209660:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077876/real 1769077876] req@ffff935002a7f800 x1855008913543424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077876.209675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913543424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077876.209681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077876.209683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077876.209685:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913543424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077876.209689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913543552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077876.209697:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077876/real 1769077876] req@ffff935002a7c000 x1855008913543552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077876.209711:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913543552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077876.209717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077876.209719:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077876.209720:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913543552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077876.209724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543680:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077876.209733:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209739:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077876/real 1769077876] req@ffff935002a7dc00 x1855008913543680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077876.209747:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913543680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077876.209752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077876.209754:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077876.209756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077876.209759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077876.209768:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077876/real 1769077876] req@ffff935002a7d180 x1855008913543808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077876.209781:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913543808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077876.209787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077876.209789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077876.209791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077876.209795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077876.209803:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077876.209808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077876/real 1769077876] req@ffff935002a7e300 x1855008913543936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077876.209815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913543936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077876.209820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077876.209821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077876.209823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913543936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077876.401135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077876.913176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077876.913176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077876.913221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077876.913223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077877.233132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077877.425151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077877.937123:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077877.937124:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077877.937153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077877.937221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077878.257180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077878.449170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077878.961170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077878.961201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077878.961220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077878.961229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077879.281126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077879.473162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077879.985203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077879.985294:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077879.986233:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077879.986402:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077880.305139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077880.497146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077881.009152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077881.009152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077881.009180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077881.009184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077881.329115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077881.329128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077881.329134:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077881.329140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077881.329146:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3587 00000100:00080000:3.0:1769077881.329162:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077881.329169:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077881.329206:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913544064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077881.329229:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077881.329231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077881.329233:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077881.329235:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3587 00000100:00080000:3.0:1769077881.329237:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077881.329240:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077881.329250:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077881.329259:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913544192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077881.329266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077881.329269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077881.329270:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077881.329272:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3587 00000100:00080000:3.0:1769077881.329276:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077881.329282:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913544064:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077881.329283:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077881.329291:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077881.329312:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913544320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077881.329324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077881.329326:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077881.329327:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077881.329330:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3587 00000100:00080000:3.0:1769077881.329333:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077881.329335:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077881.329338:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077881.329347:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913544448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077881.329352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077881.329355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077881.329356:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077881.329359:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3587 00000100:00080000:3.0:1769077881.329360:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077881.329363:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077881.329365:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077881.329372:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913544576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077881.329377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077881.329379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077881.329380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077881.329383:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3587 00000100:00080000:3.0:1769077881.329385:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077881.329387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077881.329390:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077881.329398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913544704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077881.329401:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077881.329414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077881.329416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077881.329417:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077881.329420:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3587 00000100:00080000:3.0:1769077881.329423:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077881.329425:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077881.329428:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077881.329436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913544832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077881.329466:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077881/real 1769077881] req@ffff935002a7ca80 x1855008913544064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077881.329485:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913544064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077881.329500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077881.329505:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077881.329509:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913544064:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077881.329517:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913544192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077881.329538:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077881.329550:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077881/real 1769077881] req@ffff935002a7c700 x1855008913544192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077881.329560:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913544192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077881.329568:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077881.329571:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077881.329574:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913544192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077881.329597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913544320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077881.329615:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077881.329625:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077881/real 1769077881] req@ffff935002a7c380 x1855008913544320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077881.329637:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913544320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077881.329645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077881.329649:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077881.329652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913544320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077881.329659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913544448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077881.329684:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077881.329696:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077881/real 1769077881] req@ffff935002a7f100 x1855008913544448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077881.329707:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913544448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077881.329715:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077881.329719:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077881.329722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913544448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077881.329729:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913544576:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077881.329746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077881.329756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077881/real 1769077881] req@ffff935002a7e300 x1855008913544576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077881.329768:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913544576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077881.329776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077881.329779:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077881.329782:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913544576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077881.329787:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913544704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077881.329804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077881.329815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077881/real 1769077881] req@ffff935002a7d180 x1855008913544704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077881.329826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913544704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077881.329833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077881.329836:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077881.329838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913544704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077881.329844:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913544832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077881.329861:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077881.329870:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077881/real 1769077881] req@ffff935002a7dc00 x1855008913544832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077881.329880:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913544832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077881.329888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077881.329891:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077881.329894:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913544832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077881.521151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077882.033194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077882.033195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077882.033201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077882.033243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077882.353155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077882.545196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077883.057165:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077883.057182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077883.057192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077883.057398:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077883.377216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077883.569195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077884.081152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077884.081154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077884.081181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077884.081189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077884.401157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077884.593130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077885.105175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077885.105175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077885.105175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077885.105246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077885.425166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077885.617167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077886.129176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077886.129333:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077886.129397:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077886.129442:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077886.449182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077886.449498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077886.449503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077886.449507:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077886.449514:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3592 00000100:00080000:1.0:1769077886.449523:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077886.449528:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077886.449557:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913544960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077886.449585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077886.449588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077886.449589:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077886.449593:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3592 00000100:00080000:1.0:1769077886.449595:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077886.449597:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077886.449605:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077886.449614:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913545088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077886.449621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077886.449623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077886.449624:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077886.449626:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3592 00000100:00080000:1.0:1769077886.449627:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077886.449629:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077886.449633:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077886.449638:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913545216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077886.449644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077886.449647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077886.449648:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077886.449650:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3592 00000100:00080000:1.0:1769077886.449652:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077886.449654:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077886.449656:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077886.449662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913545344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077886.449666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913544960:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077886.449671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077886.449677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077886.449679:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077886.449682:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3592 00000100:00080000:1.0:1769077886.449683:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077886.449687:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077886.449692:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077886.449702:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913545472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077886.449709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077886.449711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077886.449713:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077886.449715:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3592 00000100:00080000:1.0:1769077886.449719:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077886.449721:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077886.449725:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077886.449732:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077886.449735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913545600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077886.449748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077886.449751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077886.449752:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077886.449754:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3592 00000100:00080000:1.0:1769077886.449757:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077886.449759:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077886.449763:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077886.449767:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077886/real 1769077886] req@ffff9350100cb800 x1855008913544960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077886.449772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913545728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077886.449788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913544960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077886.449803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077886.449807:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077886.449810:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913544960:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077886.449815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545088:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077886.449830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077886.449837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077886/real 1769077886] req@ffff9350100ca300 x1855008913545088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077886.449845:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913545088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077886.449850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077886.449852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077886.449854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077886.449857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077886.449867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077886.449872:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077886/real 1769077886] req@ffff9350100c8380 x1855008913545216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077886.449883:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913545216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077886.449888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077886.449889:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077886.449891:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077886.449909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077886.449926:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077886.449945:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077886/real 1769077886] req@ffff9350100caa00 x1855008913545344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077886.449955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913545344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077886.449963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077886.449966:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077886.449969:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077886.449972:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913545472:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077886.449986:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077886.449993:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077886/real 1769077886] req@ffff9350100c9500 x1855008913545472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077886.449999:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913545472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077886.450008:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077886.450010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077886.450012:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913545472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077886.450015:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913545600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077886.450023:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077886.450028:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077886/real 1769077886] req@ffff9350100c9880 x1855008913545600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077886.450034:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913545600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077886.450038:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077886.450040:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077886.450042:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913545600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077886.450044:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913545728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077886.450082:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077886.450090:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077886/real 1769077886] req@ffff9350100c8e00 x1855008913545728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077886.450096:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913545728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077886.450101:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077886.450103:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077886.450105:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913545728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077886.641185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077887.153143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077887.153158:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077887.153169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077887.153194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077887.473144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077887.665126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077888.177137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077888.177137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077888.177137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077888.177285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077888.497166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077888.689132:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077889.201142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077889.201144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077889.201176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077889.201209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077889.521144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077889.713166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077890.225230:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077890.225230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077890.225230:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077890.225273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077890.545212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077890.737230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077891.249165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077891.249165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077891.249165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077891.249375:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077891.569208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077891.569215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077891.569220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077891.569226:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077891.569236:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3597 00000100:00080000:1.0:1769077891.569247:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077891.569254:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077891.569293:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913545856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077891.569314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077891.569317:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077891.569318:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077891.569321:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3597 00000100:00080000:1.0:1769077891.569323:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077891.569325:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077891.569334:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077891.569342:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913545984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077891.569348:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077891.569351:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077891.569352:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077891.569355:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3597 00000100:00080000:1.0:1769077891.569358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077891.569360:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077891.569364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077891.569372:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913545856:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077891.569374:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913546112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077891.569385:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077891.569387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077891.569388:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077891.569391:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3597 00000100:00080000:1.0:1769077891.569392:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077891.569394:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077891.569397:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077891.569404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913546240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077891.569410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077891.569412:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077891.569413:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077891.569416:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3597 00000100:00080000:1.0:1769077891.569417:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077891.569420:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077891.569424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077891.569434:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913546368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077891.569448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077891.569453:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077891.569454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077891.569459:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077891.569464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3597 00000100:00080000:1.0:1769077891.569468:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077891.569472:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077891.569478:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077891.569488:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913546496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077891.569494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077891.569497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077891.569498:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077891.569500:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3597 00000100:00080000:1.0:1769077891.569503:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077891.569507:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077891/real 1769077891] req@ffff9350100ca680 x1855008913545856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077891.569508:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077891.569514:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077891.569524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913546624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077891.569535:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913545856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077891.569547:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077891.569552:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077891.569556:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913545856:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077891.569563:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545984:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077891.569583:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077891.569593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077891/real 1769077891] req@ffff9350100cad80 x1855008913545984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077891.569614:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913545984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077891.569622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077891.569626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077891.569629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913545984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077891.569655:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913546112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077891.569672:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077891.569681:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077891/real 1769077891] req@ffff9350100c9c00 x1855008913546112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077891.569692:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913546112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077891.569699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077891.569702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077891.569705:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913546112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077891.569710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913546240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077891.569723:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077891.569731:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077891/real 1769077891] req@ffff9350100cb100 x1855008913546240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077891.569741:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913546240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077891.569749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077891.569752:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077891.569754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913546240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077891.569759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913546368:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077891.569773:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077891.569781:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077891/real 1769077891] req@ffff9350100c9180 x1855008913546368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077891.569792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913546368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077891.569800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077891.569803:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077891.569806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913546368:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077891.569810:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913546496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077891.569824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077891.569833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077891/real 1769077891] req@ffff9350100c8a80 x1855008913546496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077891.569843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913546496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077891.569849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077891.569852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077891.569855:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913546496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077891.569860:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913546624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077891.569874:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077891.569883:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077891/real 1769077891] req@ffff9350100c8000 x1855008913546624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077891.569899:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913546624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077891.569907:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077891.569910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077891.569912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913546624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077891.761144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077892.273105:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077892.273142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077892.273146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077892.273189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077892.593208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077892.785194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077893.297175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077893.297175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077893.297220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077893.297309:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077893.617190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077893.809139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077894.321199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077894.321205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077894.321210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077894.321260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077894.641139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077894.833209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077895.345148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077895.345168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077895.345187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077895.345215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077895.665185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077895.857176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077896.369187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077896.369187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077896.369203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077896.369243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077896.689158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077896.689162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077896.689170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077896.689175:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077896.689184:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3602 00000100:00080000:3.0:1769077896.689195:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077896.689202:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077896.689241:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913546752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077896.689267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077896.689270:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077896.689272:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077896.689275:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3602 00000100:00080000:3.0:1769077896.689277:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077896.689280:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077896.689289:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077896.689297:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913546880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077896.689303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077896.689306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077896.689307:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077896.689310:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3602 00000100:00080000:3.0:1769077896.689311:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077896.689314:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077896.689318:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077896.689324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913547008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077896.689331:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913546752:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077896.689333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077896.689338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077896.689340:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077896.689342:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3602 00000100:00080000:3.0:1769077896.689344:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077896.689346:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077896.689348:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077896.689355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913547136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077896.689361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077896.689363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077896.689364:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077896.689367:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3602 00000100:00080000:3.0:1769077896.689368:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077896.689371:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077896.689373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077896.689381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913547264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077896.689386:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077896.689388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077896.689389:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077896.689391:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3602 00000100:00080000:3.0:1769077896.689401:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077896.689403:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077896.689406:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077896.689410:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077896.689416:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913547392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077896.689422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077896.689424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077896.689425:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077896.689427:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3602 00000100:00080000:3.0:1769077896.689429:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077896.689432:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077896.689435:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077896.689441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913547520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077896.689460:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077896/real 1769077896] req@ffff935002a7c000 x1855008913546752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077896.689475:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913546752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077896.689482:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077896.689485:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077896.689488:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913546752:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077896.689493:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913546880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077896.689506:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077896.689513:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077896/real 1769077896] req@ffff935002a7f800 x1855008913546880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077896.689520:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913546880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077896.689525:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077896.689527:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077896.689529:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913546880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077896.689552:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077896.689561:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077896.689566:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077896/real 1769077896] req@ffff935002a7df80 x1855008913547008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077896.689572:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913547008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077896.689577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077896.689579:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077896.689580:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077896.689583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077896.689592:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077896.689597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077896/real 1769077896] req@ffff935002a7fb80 x1855008913547136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077896.689607:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913547136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077896.689612:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077896.689613:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077896.689615:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077896.689618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913547264:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077896.689625:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077896.689630:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077896/real 1769077896] req@ffff935002a7dc00 x1855008913547264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077896.689635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913547264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077896.689640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077896.689642:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077896.689644:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913547264:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077896.689647:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913547392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077896.689655:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077896.689660:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077896/real 1769077896] req@ffff935002a7d180 x1855008913547392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077896.689666:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913547392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077896.689674:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077896.689676:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077896.689678:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913547392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077896.689681:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913547520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077896.689697:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077896.689708:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077896/real 1769077896] req@ffff935002a7e300 x1855008913547520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077896.689719:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913547520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077896.689726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077896.689729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077896.689731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913547520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077896.881162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077897.393136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077897.393169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077897.393169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077897.393198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077897.713159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077897.905141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077898.417168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077898.417186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077898.417187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077898.417252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077898.737208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077898.929133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077899.441113:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077899.441121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077899.441125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077899.441311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077899.761182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077899.953200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077900.465172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077900.465172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077900.465172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077900.465214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077900.785179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077900.977203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077901.489167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077901.489186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077901.489188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077901.489265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077901.809163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077901.809164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077901.809176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077901.809181:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077901.809192:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3607 00000100:00080000:3.0:1769077901.809204:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077901.809213:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077901.809255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913547648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077901.809290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077901.809295:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077901.809298:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077901.809303:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3607 00000100:00080000:3.0:1769077901.809308:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077901.809313:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077901.809324:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077901.809339:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913547776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077901.809353:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913547648:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077901.809355:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077901.809366:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077901.809369:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077901.809376:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3607 00000100:00080000:3.0:1769077901.809382:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077901.809386:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077901.809392:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077901.809405:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913547904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077901.809428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077901.809434:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077901.809436:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077901.809441:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077901.809446:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3607 00000100:00080000:3.0:1769077901.809452:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077901.809457:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077901.809463:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077901.809477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913548032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077901.809489:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077901/real 1769077901] req@ffff935002a7f100 x1855008913547648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077901.809491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077901.809500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077901.809505:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077901.809516:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3607 00000100:00080000:3.0:1769077901.809525:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077901.809532:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077901.809540:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913547648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077901.809547:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077901.809552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077901.809566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913548160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077901.809570:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077901.809583:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913547648:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077901.809586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077901.809593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077901.809596:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077901.809602:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3607 00000100:00080000:3.0:1769077901.809607:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077901.809612:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077901.809618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547776:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077901.809618:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077901.809645:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913548288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077901.809659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077901.809665:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769077901.809668:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077901.809669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077901.809677:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3607 00000100:00080000:3.0:1769077901.809683:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077901.809688:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077901.809693:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077901/real 1769077901] req@ffff935002a7c380 x1855008913547776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077901.809696:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077901.809716:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913548416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077901.809721:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913547776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077901.809746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077901.809752:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077901.809756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077901.809765:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077901.809788:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077901.809801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077901/real 1769077901] req@ffff935002a7c700 x1855008913547904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077901.809815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913547904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077901.809831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077901.809835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077901.809838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913547904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077901.809846:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077901.809867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077901.809880:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077901/real 1769077901] req@ffff935002a7ca80 x1855008913548032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077901.809892:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913548032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077901.809902:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077901.809906:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077901.809910:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077901.809917:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913548160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077901.809937:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077901.809981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077901/real 1769077901] req@ffff935002a7e300 x1855008913548160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077901.809997:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913548160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077901.810007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077901.810010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077901.810015:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913548160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077901.810036:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913548288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077901.810099:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077901.810112:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077901/real 1769077901] req@ffff935002a7d180 x1855008913548288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077901.810125:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913548288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077901.810135:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077901.810138:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077901.810149:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913548288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077901.810156:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913548416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077901.810176:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077901.810188:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077901/real 1769077901] req@ffff935002a7dc00 x1855008913548416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077901.810201:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913548416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077901.810209:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077901.810213:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077901.810217:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913548416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077902.001172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077902.513164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077902.513164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077902.513165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077902.513211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077902.833157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077903.025178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077903.537167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077903.537167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077903.537175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077903.537293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077903.857161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077904.049139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077904.561177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077904.561177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077904.561177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077904.561218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077904.881175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077905.073171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077905.585168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077905.585168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077905.585171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077905.585232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077905.905143:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077906.097117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077906.609144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077906.609144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077906.609144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077906.609245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077906.929165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077906.929167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077906.929178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077906.929184:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077906.929196:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3612 00000100:00080000:1.0:1769077906.929209:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077906.929218:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077906.929267:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913548544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077906.929301:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077906.929306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077906.929308:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077906.929313:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3612 00000100:00080000:1.0:1769077906.929317:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077906.929322:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077906.929335:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077906.929354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913548672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077906.929367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077906.929372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077906.929377:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077906.929382:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913548544:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077906.929387:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3612 00000100:00080000:1.0:1769077906.929399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077906.929405:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077906.929412:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077906.929429:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913548800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077906.929440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077906.929444:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077906.929447:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077906.929452:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3612 00000100:00080000:1.0:1769077906.929467:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077906.929472:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077906.929478:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077906.929483:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077906.929497:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913548928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077906.929512:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077906.929516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077906.929519:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077906.929524:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3612 00000100:00080000:1.0:1769077906.929530:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077906.929537:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077906.929539:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9350100c9f80 x1855008913548544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077906.929547:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077906.929570:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913548544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077906.929576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913549056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077906.929595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077906.929605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077906.929610:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769077906.929615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077906.929620:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077906.929621:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913548544:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077906.929633:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3612 00000100:00080000:1.0:1769077906.929639:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077906.929645:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077906.929652:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077906.929669:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077906.929670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913549184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077906.929693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077906.929697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077906.929700:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077906.929713:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3612 00000800:00000100:0.0:1769077906.929714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077906.929727:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077906.929737:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9350100c8000 x1855008913548672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077906.929738:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077906.929756:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077906.929760:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913548672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077906.929772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077906.929777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769077906.929779:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913549312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077906.929783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077906.929805:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077906.929834:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077906.929842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9350100c8a80 x1855008913548800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077906.929855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913548800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077906.929861:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077906.929890:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077906.929893:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077906.929898:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077906.929914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077906.929922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9350100c9180 x1855008913548928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077906.929930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913548928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077906.929935:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077906.929938:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077906.929940:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913548928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077906.929944:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549056:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077906.929954:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077906.929967:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9350100cb100 x1855008913549056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077906.929980:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913549056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077906.929986:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077906.929988:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077906.929990:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077906.930005:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077906.930016:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077906.930023:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9350100c9c00 x1855008913549184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077906.930031:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913549184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077906.930036:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077906.930038:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077906.930039:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077906.930043:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077906.930097:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077906.930105:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9350100cad80 x1855008913549312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077906.930113:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913549312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077906.930122:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077906.930125:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077906.930127:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077907.121182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077907.633209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077907.633212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077907.633213:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077907.633257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077907.953205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077908.145200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077908.657128:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077908.657171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077908.657171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077908.657244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077908.977225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077909.169169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077909.681196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077909.681196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077909.681196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077909.681398:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077910.001171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077910.193183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077910.705167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077910.705167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077910.705167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077910.705213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077911.025116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077911.217159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077911.729183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077911.729183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077911.729184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077911.729246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077912.049125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077912.049142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077912.049148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077912.049152:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077912.049160:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3617 00000100:00080000:3.0:1769077912.049170:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077912.049176:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077912.049209:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913549440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077912.049234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077912.049237:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077912.049238:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077912.049241:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3617 00000100:00080000:3.0:1769077912.049244:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077912.049246:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077912.049260:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077912.049271:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913549568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077912.049275:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913549440:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077912.049285:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077912.049288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077912.049290:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077912.049294:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3617 00000100:00080000:3.0:1769077912.049297:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077912.049300:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077912.049304:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077912.049314:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913549696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077912.049325:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077912.049328:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077912.049331:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077912.049335:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3617 00000100:00080000:3.0:1769077912.049340:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077912.049344:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077912.049348:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077912.049355:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077912.049369:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913549824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077912.049380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077912.049384:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769077912.049386:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077912/real 1769077912] req@ffff935002a7fb80 x1855008913549440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077912.049387:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077912.049394:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3617 00000100:00080000:3.0:1769077912.049399:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077912.049405:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077912.049410:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077912.049419:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913549440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077912.049423:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913549952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077912.049452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077912.049454:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077912.049455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077912.049458:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077912.049462:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077912.049464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3617 00000100:00080000:3.0:1769077912.049469:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077912.049473:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913549440:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077912.049475:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077912.049479:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077912.049489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913550080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077912.049496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077912.049498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077912.049500:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077912.049502:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3617 00000100:00080000:3.0:1769077912.049505:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077912.049507:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077912.049512:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913549568:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077912.049512:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077912.049522:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913550208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077912.049546:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077912.049558:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077912/real 1769077912] req@ffff935002a7df80 x1855008913549568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077912.049568:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913549568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077912.049575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077912.049577:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077912.049580:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913549568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077912.049584:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913549696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077912.049596:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077912.049602:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077912/real 1769077912] req@ffff935002a7f800 x1855008913549696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077912.049609:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913549696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077912.049615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077912.049617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077912.049619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913549696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077912.049622:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913549824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077912.049631:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077912.049637:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077912/real 1769077912] req@ffff935002a7c000 x1855008913549824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077912.049644:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913549824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077912.049649:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077912.049651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077912.049654:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913549824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077912.049657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077912.049667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077912.049672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077912/real 1769077912] req@ffff935002a7dc00 x1855008913549952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077912.049679:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913549952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077912.049684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077912.049686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077912.049688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913549952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077912.049692:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077912.049701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077912.049706:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077912/real 1769077912] req@ffff935002a7d180 x1855008913550080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077912.049714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913550080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077912.049719:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077912.049720:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077912.049722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077912.049736:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077912.049746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077912.049752:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077912/real 1769077912] req@ffff935002a7e300 x1855008913550208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077912.049760:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913550208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077912.049765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077912.049767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077912.049769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077912.241164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077912.753161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077912.753161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077912.753161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077912.753210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077913.073145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077913.265146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077913.777173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077913.777193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077913.777198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077913.777247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077914.097128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077914.289160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077914.801133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077914.801133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077914.801133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077914.801208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077915.121171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077915.313141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077915.825169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077915.825169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077915.825222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077915.825240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077916.145163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077916.337180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077916.849142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077916.849146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077916.849152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077916.849249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077917.169168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077917.169178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077917.169183:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077917.169193:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3622 00000100:00080000:1.0:1769077917.169205:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077917.169213:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077917.169255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913550336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077917.169279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077917.169283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077917.169284:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077917.169287:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3622 00000100:00080000:1.0:1769077917.169290:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077917.169292:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077917.169302:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077917.169312:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913550464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077917.169319:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077917.169322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077917.169324:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077917.169326:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3622 00000100:00080000:1.0:1769077917.169328:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077917.169331:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077917.169335:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077917.169340:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913550336:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077917.169345:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913550592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077917.169358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077917.169362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077917.169364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077917.169369:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3622 00000100:00080000:1.0:1769077917.169374:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077917.169379:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077917.169385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077917.169400:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913550720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077917.169411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077917.169415:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077917.169417:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077917.169420:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3622 00000100:00080000:1.0:1769077917.169424:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077917.169427:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077917.169432:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077917.169443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913550848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077917.169450:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913550464:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077917.169453:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077917.169458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077917.169459:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077917.169463:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3622 00000100:00080000:1.0:1769077917.169466:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077917.169468:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077917.169472:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077917.169484:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913550976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077917.169491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077917.169494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077917.169495:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077917.169498:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3622 00000100:00080000:1.0:1769077917.169501:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077917.169507:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077917.169508:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913550592:192.168.204.121@tcp:8:kworker.0 00000008:00100000:1.0:1769077917.169510:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077917.169521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913551104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077917.169544:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913550720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077917.169555:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769077917.169567:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077917.169585:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077917.169614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913551104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077917.361188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077917.873173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077917.873186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077917.873217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077917.873261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077918.193214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077918.385207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077918.897239:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077918.897239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077918.897240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077918.897307:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077919.217168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077919.409180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077919.921171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077919.921171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077919.921189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077919.921204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077920.241187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077920.241406:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077920.241451:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077920.241470:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077920.241513:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077920.241524:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077920.241534:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077920.241545:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077920.241558:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077920.241571:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077917/real 1769077920] req@ffff9350100c9880 x1855008913550592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769077920.241571:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769077920.241604:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913550592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077920.241617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077920.241622:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077920.241626:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913550592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077920.241634:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077917/real 1769077920] req@ffff9350100c9500 x1855008913550720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077920.241677:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913550720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077920.241685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077920.241689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077920.241692:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913550720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077920.241700:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077917/real 1769077920] req@ffff9350100caa00 x1855008913550848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077920.241713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913550848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077920.241722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077920.241725:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077920.241730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077920.241735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077917/real 1769077920] req@ffff9350100c8380 x1855008913550976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077920.241746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913550976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077920.241751:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077920.241753:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077920.241756:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913550976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077920.241760:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077917/real 1769077920] req@ffff9350100ca680 x1855008913550336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077920.241771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913550336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077920.241778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077920.241781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077920.241784:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913550336:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077920.241791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077917/real 1769077920] req@ffff9350100c8e00 x1855008913550464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077920.241802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913550464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077920.241810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077920.241813:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077920.241816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913550464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077920.241853:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077917/real 1769077920] req@ffff9350100ca300 x1855008913551104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077920.241864:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913551104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077920.241872:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077920.241876:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077920.241879:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913551104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077920.433176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077920.945193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077920.945193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077920.945319:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077920.945365:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077921.265142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077921.457195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077921.969134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077921.969203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077921.969226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077921.969234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077922.289170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077922.289184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077922.289191:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077922.289201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077922.289205:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3627 00000100:00080000:1.0:1769077922.289222:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077922.289232:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077922.289277:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913551232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077922.289307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077922.289311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077922.289313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077922.289318:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3627 00000100:00080000:1.0:1769077922.289321:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077922.289326:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077922.289339:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077922.289357:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913551360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077922.289359:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913551232:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077922.289377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077922.289381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077922.289383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077922.289389:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3627 00000100:00080000:1.0:1769077922.289393:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077922.289398:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077922.289405:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077922.289416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913551488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077922.289434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077922.289438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077922.289440:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077922.289445:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3627 00000100:00080000:1.0:1769077922.289450:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077922.289454:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077922.289460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077922.289465:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077922.289483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913551616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077922.289494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077922.289498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077922.289500:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077922.289505:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3627 00000100:00080000:1.0:1769077922.289510:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077922.289515:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077922.289521:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077922.289523:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077922/real 1769077922] req@ffff9350100cb800 x1855008913551232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077922.289536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913551744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077922.289548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077922.289553:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913551232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077922.289554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077922.289558:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077922.289568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3627 00000100:00080000:0.0:1769077922.289575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077922.289575:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077922.289583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077922.289587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769077922.289590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077922.289595:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913551232:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077922.289621:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913551872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077922.289632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077922.289636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077922.289638:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077922.289645:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913551360:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077922.289645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3627 00000100:00080000:1.0:1769077922.289651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077922.289657:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077922.289662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077922.289678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913552000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077922.289687:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077922.289708:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077922/real 1769077922] req@ffff9350100ca300 x1855008913551360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077922.289733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913551360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077922.289744:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077922.289749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077922.289753:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913551360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077922.289760:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913551488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077922.289785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077922.289800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077922/real 1769077922] req@ffff9350100c8e00 x1855008913551488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077922.289813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913551488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077922.289822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077922.289826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077922.289837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913551488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077922.289844:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913551616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077922.289898:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077922.289913:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077922/real 1769077922] req@ffff9350100ca680 x1855008913551616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077922.289926:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913551616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077922.289942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077922.289946:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077922.289950:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913551616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077922.289957:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913551744:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077922.289979:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077922.290001:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077922/real 1769077922] req@ffff9350100c8380 x1855008913551744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077922.290017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913551744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077922.290028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077922.290032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077922.290036:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913551744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077922.290042:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913551872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077922.290102:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077922.290117:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077922/real 1769077922] req@ffff9350100caa00 x1855008913551872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077922.290135:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913551872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077922.290162:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077922.290166:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077922.290173:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913551872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077922.290207:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077922.290239:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077922.290253:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077922/real 1769077922] req@ffff9350100c9500 x1855008913552000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077922.290268:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913552000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077922.290277:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077922.290281:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077922.290285:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077922.481182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077922.993201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077922.993216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077922.993235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077922.993260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077923.313243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077923.505153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077924.017130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077924.017131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077924.017145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077924.017173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077924.337166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077924.529154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077925.041177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077925.041181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077925.041211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077925.041217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077925.361188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077925.553152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077926.065136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077926.065139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077926.065158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077926.065297:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077926.385200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077926.577188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077927.089183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077927.089183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077927.089218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077927.089226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:2.0:1769077927.409217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077927.409220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077927.409233:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077927.409240:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077927.409251:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3633 00000100:00080000:2.0:1769077927.409263:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077927.409271:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077927.409325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913552128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077927.409353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077927.409356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077927.409358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077927.409361:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3633 00000100:00080000:2.0:1769077927.409364:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077927.409369:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077927.409379:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077927.409390:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913552256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077927.409398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077927.409401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077927.409403:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077927.409407:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3633 00000100:00080000:2.0:1769077927.409409:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077927.409414:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077927.409418:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913552128:192.168.204.121@tcp:250:kworker.0 00000008:00100000:2.0:1769077927.409419:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077927.409430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913552384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077927.409438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077927.409440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077927.409442:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077927.409444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3633 00000100:00080000:2.0:1769077927.409447:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077927.409449:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077927.409452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077927.409458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913552512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077927.409465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077927.409469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077927.409470:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077927.409473:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3633 00000100:00080000:2.0:1769077927.409475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077927.409478:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077927.409480:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077927.409488:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913552640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077927.409494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077927.409496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077927.409497:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077927.409501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3633 00000800:00000100:0.0:1769077927.409507:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077927.409507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077927.409516:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077927.409532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077927.409547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913552768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077927.409558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077927.409562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077927.409564:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077927.409568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3633 00000100:00080000:2.0:1769077927.409573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077927.409579:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077927.409580:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077927/real 1769077927] req@ffff935002a63100 x1855008913552128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077927.409584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077927.409599:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913552896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077927.409610:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913552128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077927.409625:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077927.409630:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077927.409636:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913552128:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077927.409644:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913552256:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077927.409669:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077927.409683:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077927/real 1769077927] req@ffff935002a61500 x1855008913552256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077927.409697:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913552256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077927.409707:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077927.409710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077927.409714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913552256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077927.409743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913552384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077927.409764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077927.409775:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077927/real 1769077927] req@ffff935002a60000 x1855008913552384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077927.409788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913552384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077927.409798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077927.409802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077927.409806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913552384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077927.409813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913552512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077927.409832:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077927.409844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077927/real 1769077927] req@ffff935002a62a00 x1855008913552512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077927.409857:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913552512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077927.409867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077927.409870:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077927.409874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913552512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077927.409881:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552640:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077927.409900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077927.409911:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077927/real 1769077927] req@ffff935002a61c00 x1855008913552640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077927.409924:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913552640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077927.409933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077927.409937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077927.409940:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077927.409947:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077927.409966:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077927.409978:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077927/real 1769077927] req@ffff935002a61f80 x1855008913552768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077927.409990:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913552768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077927.409999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077927.410002:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077927.410005:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077927.410011:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077927.410047:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077927.410088:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077927/real 1769077927] req@ffff935002a60a80 x1855008913552896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077927.410102:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913552896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077927.410112:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077927.410116:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077927.410119:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913552896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077927.601116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077928.113128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077928.113150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077928.113151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077928.113233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077928.433203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077928.625176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077929.137202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077929.137203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077929.137247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077929.137260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077929.457169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077929.649192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077930.161174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077930.161174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077930.161199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077930.161229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077930.481181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077930.673181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077931.185172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077931.185172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077931.185172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077931.185222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077931.505161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077931.697219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077932.209210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077932.209210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077932.209213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077932.209244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077932.529194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077932.529202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077932.529210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077932.529214:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077932.529223:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3638 00000100:00080000:0.0:1769077932.529234:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077932.529239:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077932.529274:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913553024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077932.529301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077932.529304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077932.529305:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077932.529308:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3638 00000100:00080000:0.0:1769077932.529311:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077932.529314:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077932.529323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077932.529332:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913553152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077932.529341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077932.529343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077932.529344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077932.529348:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3638 00000100:00080000:0.0:1769077932.529350:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077932.529356:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077932.529361:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077932.529362:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913553024:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077932.529377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913553280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077932.529384:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077932.529386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077932.529387:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077932.529391:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3638 00000100:00080000:0.0:1769077932.529393:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077932.529396:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077932.529401:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077932.529415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913553408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077932.529422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077932.529425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077932.529427:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077932.529429:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3638 00000100:00080000:0.0:1769077932.529448:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077932.529452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077932.529455:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077932.529467:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913553536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077932.529483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077932.529491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077932.529497:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077932.529497:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769077932.529511:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3638 00000100:00080000:0.0:1769077932.529519:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077932.529524:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077932.529530:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077932.529546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913553664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077932.529555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077932.529556:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077932/real 1769077932] req@ffff9350103f9f80 x1855008913553024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077932.529566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077932.529573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077932.529585:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3638 00000100:00080000:0.0:1769077932.529591:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077932.529597:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077932.529601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077932.529601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913553024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077932.529627:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077932.529628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913553792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077932.529642:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077932.529656:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913553024:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077932.529697:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913553152:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077932.529729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077932.529745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077932/real 1769077932] req@ffff9350103f9c00 x1855008913553152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077932.529760:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913553152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077932.529770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077932.529774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077932.529778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913553152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077932.529786:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913553280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077932.529807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077932.529819:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077932/real 1769077932] req@ffff9350103fbb80 x1855008913553280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077932.529840:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913553280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077932.529850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077932.529854:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077932.529858:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913553280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077932.529865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913553408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077932.529886:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077932.529898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077932/real 1769077932] req@ffff9350103f9500 x1855008913553408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077932.529922:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913553408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077932.529932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077932.529936:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077932.529940:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913553408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077932.529948:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913553536:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077932.529978:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077932.529991:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077932/real 1769077932] req@ffff9350103f9180 x1855008913553536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077932.530004:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913553536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077932.530019:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077932.530023:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077932.530027:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913553536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077932.530034:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913553664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077932.530086:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077932.530099:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077932/real 1769077932] req@ffff9350103f8380 x1855008913553664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077932.530112:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913553664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077932.530122:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077932.530125:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077932.530129:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913553664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077932.530136:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913553792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077932.530158:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077932.530170:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077932/real 1769077932] req@ffff9350103fb100 x1855008913553792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077932.530183:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913553792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077932.530191:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077932.530195:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077932.530199:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913553792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077932.721161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077933.233145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077933.233145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077933.233159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077933.233190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077933.553144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077933.745184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077934.257173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077934.257179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077934.257185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077934.257242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077934.577165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077934.769220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077935.281179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077935.281179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077935.281183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077935.281240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077935.601187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077935.793196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077936.305141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077936.305172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077936.305216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077936.305265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077936.625204:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077936.817200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077937.329138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077937.329139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077937.329145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077937.329170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077937.649204:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077937.649212:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077937.649223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077937.649227:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077937.649239:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3643 00000100:00080000:2.0:1769077937.649252:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077937.649260:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077937.649309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913553920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077937.649334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077937.649337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077937.649338:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077937.649341:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3643 00000100:00080000:2.0:1769077937.649344:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077937.649361:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077937.649373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077937.649389:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913553920:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077937.649392:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913554048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077937.649404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077937.649409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077937.649410:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077937.649413:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3643 00000100:00080000:2.0:1769077937.649417:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077937.649419:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077937.649423:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077937.649433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913554176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077937.649444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077937.649447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077937.649448:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077937.649452:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3643 00000100:00080000:2.0:1769077937.649455:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077937.649457:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077937.649460:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077937.649467:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077937.649470:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913554304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077937.649478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077937.649481:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077937.649482:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077937.649486:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3643 00000100:00080000:2.0:1769077937.649494:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077937.649497:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077937.649501:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077937.649514:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077937/real 1769077937] req@ffff935002a62680 x1855008913553920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077937.649537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913553920/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077937.649562:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913554432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077937.649570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077937.649573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077937.649574:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077937.649576:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3643 00000100:00080000:2.0:1769077937.649579:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077937.649581:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077937.649586:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077937.649594:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769077937.649598:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913554560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077937.649602:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077937.649611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913553920:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077937.649613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077937.649620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077937.649621:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077937.649625:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3643 00000100:00100000:0.0:1769077937.649644:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554048:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077937.649659:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077937.649665:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077937.649672:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077937.649680:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077937.649690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913554688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077937.649741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077937/real 1769077937] req@ffff935002a61180 x1855008913554048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077937.649758:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913554048/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077937.649769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077937.649773:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077937.649777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077937.649784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077937.649810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077937.649823:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077937/real 1769077937] req@ffff935002a61880 x1855008913554176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077937.649838:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913554176/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077937.649848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077937.649853:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077937.649858:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077937.649865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077937.649899:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077937.649914:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077937/real 1769077937] req@ffff935002a62300 x1855008913554304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077937.649930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913554304/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077937.649949:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077937.649953:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077937.649957:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077937.649965:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913554432:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077937.649991:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077937.650004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077937/real 1769077937] req@ffff935002a60a80 x1855008913554432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077937.650014:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913554432/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077937.650022:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077937.650024:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077937.650027:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913554432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077937.650034:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913554560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077937.650083:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077937.650097:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077937/real 1769077937] req@ffff935002a61f80 x1855008913554560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077937.650106:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913554560/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077937.650123:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077937.650127:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077937.650130:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913554560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077937.650154:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913554688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077937.650175:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077937.650185:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077937/real 1769077937] req@ffff935002a61c00 x1855008913554688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077937.650194:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913554688/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077937.650203:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077937.650206:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077937.650210:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913554688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077937.841194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077938.353129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077938.353194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077938.353194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077938.353236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077938.673176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077938.865154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077939.377185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077939.377206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077939.377207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077939.377289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077939.697187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077939.889204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077940.401130:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077940.401172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077940.401176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077940.401192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077940.721215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077940.913166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077941.425181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077941.425187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077941.425228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077941.425327:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077941.745193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077941.937127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077942.449193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077942.449193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077942.449210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077942.449249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077942.769173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077942.769186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077942.769197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077942.769202:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077942.769215:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3648 00000100:00080000:2.0:1769077942.769229:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077942.769237:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077942.769306:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913554816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077942.769331:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077942.769334:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077942.769336:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077942.769340:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3648 00000100:00080000:2.0:1769077942.769343:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077942.769347:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077942.769359:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077942.769366:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913554816:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077942.769375:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913554944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077942.769384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077942.769387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077942.769388:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077942.769391:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3648 00000100:00080000:2.0:1769077942.769395:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077942.769397:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077942.769401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077942.769411:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913555072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077942.769419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077942.769422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077942.769423:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077942.769425:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3648 00000100:00080000:2.0:1769077942.769428:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077942.769431:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077942.769433:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077942.769440:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913555200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077942.769445:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077942.769449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077942.769451:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077942.769454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769077942.769457:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3648 00000100:00080000:2.0:1769077942.769461:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077942.769465:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077942.769468:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077942.769477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913555328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077942.769489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077942.769491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077942.769493:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077942.769495:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3648 00000100:00080000:2.0:1769077942.769497:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077942.769500:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077942.769504:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077942.769511:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077942/real 1769077942] req@ffff935002a62a00 x1855008913554816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077942.769521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913555456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077942.769529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077942.769532:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913554816/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077942.769533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077942.769537:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077942.769544:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3648 00000100:00080000:2.0:1769077942.769549:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077942.769553:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077942.769555:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769077942.769557:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077942.769562:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077942.769570:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913555584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077942.769571:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913554816:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769077942.769599:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554944:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077942.769620:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077942.769629:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077942/real 1769077942] req@ffff935002a60000 x1855008913554944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077942.769643:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913554944/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077942.769652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077942.769656:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077942.769660:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913554944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077942.769667:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077942.769689:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077942.769700:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077942/real 1769077942] req@ffff935002a61500 x1855008913555072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077942.769713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913555072/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077942.769719:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077942.769722:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077942.769724:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077942.769729:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077942.769744:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077942.769751:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077942/real 1769077942] req@ffff935002a63100 x1855008913555200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077942.769769:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913555200/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077942.769775:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077942.769777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077942.769779:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077942.769783:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913555328:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077942.769795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077942.769802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077942/real 1769077942] req@ffff935002a61c00 x1855008913555328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077942.769809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913555328/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077942.769814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077942.769817:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077942.769819:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913555328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077942.769822:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913555456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077942.769831:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077942.769837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077942/real 1769077942] req@ffff935002a61f80 x1855008913555456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077942.769852:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913555456/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077942.769858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077942.769860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077942.769862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913555456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077942.769865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913555584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077942.769875:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077942.769882:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077942/real 1769077942] req@ffff935002a60a80 x1855008913555584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077942.769889:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913555584/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077942.769895:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077942.769896:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077942.769898:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913555584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077942.961161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077943.473125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077943.473126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077943.473131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077943.473215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077943.793182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077943.985201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077944.497182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077944.497182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077944.497182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077944.497219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077944.817173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077945.009184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077945.521177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077945.521177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077945.521178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077945.521228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077945.841151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077946.033168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077946.545156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077946.545156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077946.545157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077946.545249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077946.865129:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077947.057241:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077947.569169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077947.569169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077947.569169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077947.569221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077947.889141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077947.889151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1769077947.889155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077947.889156:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077947.889170:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3653 00000100:00080000:1.0:1769077947.889193:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077947.889202:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077947.889242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913555712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077947.889271:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077947.889276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077947.889278:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077947.889283:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3653 00000100:00080000:1.0:1769077947.889287:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077947.889292:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077947.889301:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077947.889313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913555840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077947.889322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077947.889326:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077947.889329:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077947.889333:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913555712:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077947.889336:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3653 00000100:00080000:1.0:1769077947.889343:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077947.889350:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077947.889356:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077947.889381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913555968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077947.889392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077947.889395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077947.889396:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077947.889400:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3653 00000100:00080000:1.0:1769077947.889404:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077947.889409:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769077947.889414:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:1.0:1769077947.889416:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077947.889431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913556096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077947.889443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077947.889447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077947.889449:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077947.889453:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3653 00000100:00080000:1.0:1769077947.889458:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077947.889462:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077947.889467:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077947.889472:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff9350100c9880 x1855008913555712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077947.889481:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913556224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077947.889498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077947.889504:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913555712/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077947.889504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077947.889509:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077947.889518:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3653 00000100:00080000:1.0:1769077947.889525:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077947.889527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077947.889531:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077947.889537:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769077947.889538:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077947.889546:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913555712:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077947.889552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913556352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077947.889568:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077947.889570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077947.889572:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077947.889577:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555840:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077947.889578:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3653 00000100:00080000:1.0:1769077947.889582:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077947.889587:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077947.889590:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077947.889599:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913556480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077947.889622:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077947.889640:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff9350100cad80 x1855008913555840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077947.889654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913555840/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077947.889664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077947.889669:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077947.889673:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555840:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077947.889681:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077947.889704:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077947.889716:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff9350100c9c00 x1855008913555968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077947.889737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913555968/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077947.889747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077947.889750:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077947.889754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913555968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077947.889761:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077947.889794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077947.889807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff9350100cb100 x1855008913556096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077947.889820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913556096/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077947.889830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077947.889834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077947.889838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077947.889844:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913556224:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077947.889874:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077947.889886:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff9350100c9180 x1855008913556224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077947.889899:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913556224/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077947.889915:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077947.889919:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077947.889922:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913556224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077947.889929:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913556352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077947.889950:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077947.889961:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff9350100c8a80 x1855008913556352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077947.889973:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913556352/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077947.889983:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077947.889986:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077947.889990:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913556352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077947.890011:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913556480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077947.890032:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077947.890043:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff9350100c8000 x1855008913556480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077947.890090:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913556480/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077947.890099:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077947.890102:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077947.890106:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913556480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077948.081146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077948.593182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077948.593183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077948.593200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077948.593231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077948.913149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077949.105196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077949.617140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077949.617140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077949.617168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077949.617216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077949.937188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077950.129130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077950.641200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077950.641200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077950.641204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077950.641252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077950.961165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077951.153263:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077951.665194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077951.665194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077951.665220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077951.665275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077951.985179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077952.177179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077952.689176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077952.689177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077952.689179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077952.689240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077953.009205:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077953.009208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077953.009218:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077953.009223:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077953.009234:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3658 00000100:00080000:1.0:1769077953.009250:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077953.009260:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077953.009311:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913556608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077953.009337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077953.009340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077953.009341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077953.009344:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3658 00000100:00080000:1.0:1769077953.009348:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077953.009352:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077953.009363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077953.009372:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913556608:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077953.009378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913556736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077953.009389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077953.009393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077953.009395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077953.009398:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3658 00000100:00080000:1.0:1769077953.009400:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077953.009403:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077953.009407:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077953.009421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913556864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077953.009429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077953.009432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077953.009433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077953.009436:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3658 00000100:00080000:1.0:1769077953.009439:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077953.009441:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077953.009444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077953.009450:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077953.009454:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913556992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077953.009462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077953.009465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077953.009467:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077953.009472:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3658 00000100:00080000:1.0:1769077953.009475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077953.009480:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077953.009485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077953.009500:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077953/real 1769077953] req@ffff9350100c9f80 x1855008913556608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077953.009502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913557120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077953.009533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077953.009536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077953.009538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077953.009541:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913556608/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077953.009544:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3658 00000100:00080000:1.0:1769077953.009551:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077953.009557:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077953.009560:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769077953.009562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077953.009567:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077953.009571:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913556608:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077953.009576:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913557248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077953.009584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077953.009586:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077953.009588:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077953.009590:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3658 00000100:00080000:1.0:1769077953.009592:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077953.009594:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077953.009597:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077953.009602:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769077953.009606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913557376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077953.009636:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077953.009652:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077953/real 1769077953] req@ffff9350100c8000 x1855008913556736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077953.009672:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913556736/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077953.009679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077953.009681:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077953.009684:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077953.009689:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077953.009705:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077953.009712:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077953/real 1769077953] req@ffff9350100c8a80 x1855008913556864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077953.009720:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913556864/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077953.009726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077953.009728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077953.009730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077953.009733:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077953.009742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077953.009748:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077953/real 1769077953] req@ffff9350100c9180 x1855008913556992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077953.009755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913556992/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077953.009765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077953.009767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077953.009769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913556992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077953.009772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913557120:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077953.009781:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077953.009787:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077953/real 1769077953] req@ffff9350100cb100 x1855008913557120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077953.009793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913557120/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077953.009799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077953.009801:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077953.009802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913557120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077953.009806:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913557248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077953.009815:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077953.009820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077953/real 1769077953] req@ffff9350100c9c00 x1855008913557248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077953.009826:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913557248/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077953.009832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077953.009834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077953.009837:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913557248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077953.009852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913557376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077953.009862:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077953.009868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077953/real 1769077953] req@ffff9350100cad80 x1855008913557376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077953.009876:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913557376/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077953.009881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077953.009883:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077953.009885:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913557376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077953.201209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077953.713133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077953.713143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077953.713189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077953.713197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077954.033158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077954.225124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077954.737141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077954.737150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077954.737175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077954.737189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077955.057174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077955.249165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077955.761191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077955.761192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077955.761242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077955.761311:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077956.081138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077956.273149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077956.785164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077956.785165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077956.785175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077956.785230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077957.105186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077957.297185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077957.809207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077957.809207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077957.809208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077957.809248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077958.129144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077958.129155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1769077958.129156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077958.129160:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077958.129171:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3663 00000100:00080000:1.0:1769077958.129183:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077958.129190:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077958.129231:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913557504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077958.129255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077958.129258:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077958.129260:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077958.129264:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3663 00000100:00080000:1.0:1769077958.129267:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077958.129270:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077958.129281:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077958.129289:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913557504:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077958.129292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913557632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077958.129305:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077958.129310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077958.129312:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077958.129317:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3663 00000100:00080000:1.0:1769077958.129331:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077958.129336:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077958.129342:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077958.129355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913557760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077958.129362:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077958.129377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077958.129381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077958.129383:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077958.129389:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3663 00000100:00080000:1.0:1769077958.129393:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077958.129398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077958.129403:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077958.129411:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077958/real 1769077958] req@ffff9350100c9880 x1855008913557504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077958.129418:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913557888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077958.129446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077958.129451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077958.129453:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077958.129457:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913557504/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077958.129462:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3663 00000100:00080000:1.0:1769077958.129470:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077958.129473:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077958.129478:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077958.129488:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077958.129493:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913557504:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769077958.129495:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077958.129513:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913558016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077958.129521:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913557632:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077958.129524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077958.129530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077958.129532:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077958.129536:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3663 00000100:00080000:1.0:1769077958.129540:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077958.129543:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077958.129548:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077958.129560:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077958.129560:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913558144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077958.129571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077958.129574:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077958.129576:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077958.129579:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3663 00000100:00080000:1.0:1769077958.129582:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077958.129584:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077958/real 1769077958] req@ffff9350100c9500 x1855008913557632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077958.129587:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077958.129593:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077958.129603:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913558272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077958.129610:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913557632/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077958.129622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077958.129626:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077958.129631:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913557632:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077958.129638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913557760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077958.129658:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077958.129668:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077958/real 1769077958] req@ffff9350100caa00 x1855008913557760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077958.129680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913557760/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077958.129693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077958.129697:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077958.129700:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913557760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077958.129706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913557888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077958.129734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077958.129743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077958/real 1769077958] req@ffff9350100c8380 x1855008913557888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077958.129751:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913557888/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077958.129757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077958.129759:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077958.129761:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913557888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077958.129774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558016:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077958.129785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077958.129792:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077958/real 1769077958] req@ffff9350100ca680 x1855008913558016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077958.129800:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913558016/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077958.129805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077958.129806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077958.129809:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077958.129813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077958.129822:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077958.129828:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077958/real 1769077958] req@ffff9350100c8e00 x1855008913558144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077958.129835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913558144/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077958.129840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077958.129842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077958.129844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077958.129847:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077958.129859:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077958.129869:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077958/real 1769077958] req@ffff9350100ca300 x1855008913558272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077958.129882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913558272/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077958.129891:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077958.129895:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077958.129899:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077958.321179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077958.833144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077958.833148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077958.833151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077958.833227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077959.153209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077959.345169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077959.857160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077959.857160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077959.857167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077959.857189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077960.177095:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077960.369149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077960.881123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077960.881123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077960.881132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077960.881159:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077961.201161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077961.393160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077961.905163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077961.905171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077961.905226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077961.905252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077962.225133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077962.417164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077962.929151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077962.929188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077962.929283:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077962.929301:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077963.249158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769077963.249196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077963.249208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077963.249213:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077963.249224:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3668 00000100:00080000:2.0:1769077963.249236:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077963.249244:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077963.249283:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913558400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077963.249313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077963.249316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077963.249318:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077963.249321:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3668 00000100:00080000:2.0:1769077963.249324:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077963.249328:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077963.249338:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077963.249354:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913558528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077963.249368:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913558400:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077963.249381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077963.249385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077963.249388:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077963.249393:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3668 00000100:00080000:2.0:1769077963.249398:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077963.249402:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077963.249408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077963.249420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913558656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077963.249431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077963.249434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077963.249436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077963.249440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3668 00000100:00080000:2.0:1769077963.249444:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077963.249448:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077963.249453:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077963.249459:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077963.249466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913558784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077963.249478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077963.249481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077963.249484:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077963.249488:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3668 00000100:00080000:2.0:1769077963.249492:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077963.249497:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077963.249502:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077963.249506:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077963/real 1769077963] req@ffff935002a62300 x1855008913558400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077963.249518:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913558912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077963.249525:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913558400/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077963.249536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077963.249540:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077963.249540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077963.249545:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077963.249549:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769077963.249553:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3668 00000100:00100000:0.0:1769077963.249557:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913558400:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769077963.249559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077963.249567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077963.249572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077963.249582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913559040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077963.249585:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913558528:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769077963.249597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077963.249601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077963.249603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077963.249606:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3668 00000100:00080000:2.0:1769077963.249609:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077963.249612:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769077963.249616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077963.249621:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077963.249627:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913559168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077963.249636:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077963/real 1769077963] req@ffff935002a61880 x1855008913558528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077963.249660:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913558528/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077963.249667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077963.249670:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077963.249674:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913558528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077963.249679:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913558656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077963.249692:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077963.249699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077963/real 1769077963] req@ffff935002a61180 x1855008913558656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077963.249707:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913558656/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077963.249712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077963.249713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077963.249716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913558656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077963.249720:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913558784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077963.249729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077963.249735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077963/real 1769077963] req@ffff935002a62680 x1855008913558784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077963.249743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913558784/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077963.249748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077963.249749:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077963.249751:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913558784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077963.249754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558912:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077963.249764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077963.249770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077963/real 1769077963] req@ffff935002a60a80 x1855008913558912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077963.249788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913558912/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077963.249793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077963.249795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077963.249797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913558912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077963.249808:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077963.249818:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077963.249824:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077963/real 1769077963] req@ffff935002a61f80 x1855008913559040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077963.249832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913559040/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077963.249837:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077963.249839:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077963.249840:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077963.249844:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077963.249853:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077963.249859:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077963/real 1769077963] req@ffff935002a61c00 x1855008913559168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077963.249866:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913559168/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077963.249882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077963.249884:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077963.249887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077963.441204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077963.953177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077963.953183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077963.953200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077963.953211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077964.273172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077964.465130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077964.977144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077964.977144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077964.977158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077964.977193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077965.297145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077965.489162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077966.001146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077966.001159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077966.001185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077966.001193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077966.321169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077966.513126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077967.025188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077967.025188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077967.025205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077967.025252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077967.345161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077967.537170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077968.049195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077968.049203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077968.049253:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077968.049271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077968.369168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077968.369186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077968.369193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077968.369197:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077968.369207:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3674 00000100:00080000:1.0:1769077968.369220:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077968.369229:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077968.369277:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913559296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077968.369302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077968.369305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077968.369306:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077968.369310:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3674 00000100:00080000:1.0:1769077968.369314:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077968.369317:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077968.369328:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077968.369335:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913559296:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077968.369343:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913559424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077968.369356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077968.369358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077968.369360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077968.369363:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3674 00000100:00080000:1.0:1769077968.369366:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077968.369368:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077968.369372:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077968.369380:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913559552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077968.369386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077968.369389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077968.369390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077968.369393:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3674 00000100:00080000:1.0:1769077968.369396:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077968.369398:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077968.369401:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077968.369406:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077968.369410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913559680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077968.369417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077968.369419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077968.369421:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077968.369424:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3674 00000100:00080000:1.0:1769077968.369428:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077968.369430:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077968.369433:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077968.369441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913559808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077968.369446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077968.369449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077968.369450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077968.369453:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3674 00000100:00080000:1.0:1769077968.369456:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077968.369457:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077968/real 1769077968] req@ffff9350100cb800 x1855008913559296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077968.369463:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077968.369472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077968.369482:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913559296/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077968.369490:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913559936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077968.369494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077968.369502:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077968.369509:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913559296:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077968.369513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077968.369518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077968.369521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077968.369527:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3674 00000100:00080000:1.0:1769077968.369532:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077968.369536:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913559424:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077968.369540:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077968.369547:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077968.369558:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077968.369565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913560064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077968.369570:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077968/real 1769077968] req@ffff9350100ca300 x1855008913559424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077968.369596:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913559424/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077968.369606:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077968.369609:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077968.369611:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913559424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077968.369618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913559552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077968.369630:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077968.369636:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077968/real 1769077968] req@ffff9350100c8e00 x1855008913559552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077968.369644:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913559552/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077968.369649:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077968.369651:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077968.369653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913559552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077968.369656:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913559680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077968.369665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077968.369670:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077968/real 1769077968] req@ffff9350100ca680 x1855008913559680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077968.369682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913559680/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077968.369687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077968.369689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077968.369691:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913559680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077968.369694:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559808:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077968.369703:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077968.369709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077968/real 1769077968] req@ffff9350100c8380 x1855008913559808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077968.369716:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913559808/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077968.369722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077968.369723:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077968.369725:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077968.369728:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077968.369737:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077968.369743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077968/real 1769077968] req@ffff9350100caa00 x1855008913559936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077968.369750:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913559936/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077968.369759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077968.369761:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077968.369763:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913559936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077968.369776:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077968.369785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077968.369791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077968/real 1769077968] req@ffff9350100c9500 x1855008913560064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077968.369798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913560064/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077968.369803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077968.369805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077968.369807:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077968.561198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077969.073192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077969.073271:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077969.073285:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077969.073305:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077969.393185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077969.585140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077970.097145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077970.097145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077970.097155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077970.097211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077970.417174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077970.609206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077971.121158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077971.121160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077971.121187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077971.121198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077971.441124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077971.633173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077972.145207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077972.145207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077972.145207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769077972.145325:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077972.465203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077972.657161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077973.169119:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077973.169150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077973.169171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077973.169187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077973.489156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077973.489158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077973.489173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077973.489178:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077973.489189:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3679 00000100:00080000:1.0:1769077973.489200:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077973.489206:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077973.489241:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913560192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077973.489266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077973.489269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077973.489270:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077973.489274:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3679 00000100:00080000:1.0:1769077973.489279:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077973.489282:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077973.489292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077973.489308:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913560192:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077973.489308:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913560320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077973.489333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077973.489335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077973.489336:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077973.489340:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3679 00000100:00080000:1.0:1769077973.489343:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077973.489346:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077973.489351:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769077973.489363:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913560448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077973.489370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077973.489372:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077973.489374:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077973.489376:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3679 00000100:00080000:1.0:1769077973.489390:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077973.489394:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077973.489399:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077973.489408:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077973.489415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913560576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077973.489430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077973.489434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077973.489437:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077973.489442:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3679 00000100:00080000:1.0:1769077973.489447:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077973.489453:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077973.489460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077973.489464:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077973/real 1769077973] req@ffff9350100c9880 x1855008913560192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077973.489478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913560704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077973.489487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077973.489491:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913560192/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077973.489492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077973.489496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077973.489507:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3679 00000100:00080000:1.0:1769077973.489516:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077973.489529:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077973.489534:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769077973.489535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769077973.489548:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769077973.489550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913560832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077973.489568:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913560192:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077973.489568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077973.489576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077973.489581:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077973.489586:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3679 00000100:00080000:1.0:1769077973.489588:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077973.489591:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077973.489594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077973.489602:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913560960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077973.489612:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913560320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077973.489638:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077973.489649:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077973/real 1769077973] req@ffff9350100cad80 x1855008913560320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077973.489658:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913560320/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077973.489664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077973.489667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077973.489669:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913560320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077973.489674:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913560448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077973.489693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077973.489703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077973/real 1769077973] req@ffff9350100c9c00 x1855008913560448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077973.489723:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913560448/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077973.489732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077973.489736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077973.489740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913560448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077973.489768:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913560576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077973.489786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077973.489794:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077973/real 1769077973] req@ffff9350100cb100 x1855008913560576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077973.489802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913560576/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077973.489808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077973.489810:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077973.489812:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913560576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077973.489816:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560704:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077973.489827:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077973.489833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077973/real 1769077973] req@ffff9350100c9180 x1855008913560704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077973.489840:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913560704/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077973.489850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077973.489852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077973.489854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077973.489859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077973.489877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077973.489888:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077973/real 1769077973] req@ffff9350100c8a80 x1855008913560832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077973.489901:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913560832/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077973.489910:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077973.489913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077973.489916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077973.489935:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077973.489951:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077973.489959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077973/real 1769077973] req@ffff9350100c8000 x1855008913560960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077973.489967:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913560960/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077973.489972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077973.489974:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077973.489976:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913560960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077973.681167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077974.193138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077974.193168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077974.193195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077974.193218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077974.513352:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077974.705142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077975.217119:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077975.217126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077975.217148:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077975.217150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077975.537126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077975.729195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769077976.241177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077976.241177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077976.241178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077976.241254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077976.561159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077976.753154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077977.265152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077977.265152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077977.265178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077977.265235:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077977.585148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077977.777193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077978.289158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077978.289167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077978.289199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077978.289208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077978.609160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769077978.609203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077978.609217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077978.609222:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077978.609234:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3684 00000100:00080000:1.0:1769077978.609248:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077978.609257:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769077978.609305:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913561088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077978.609340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077978.609345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077978.609347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077978.609352:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3684 00000100:00080000:1.0:1769077978.609358:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077978.609364:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077978.609377:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077978.609384:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913561088:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769077978.609401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913561216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077978.609418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077978.609423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077978.609425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077978.609431:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3684 00000100:00080000:1.0:1769077978.609448:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077978.609453:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077978.609460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769077978.609461:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077978.609484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913561344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077978.609497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077978.609501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077978.609503:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077978.609507:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077978/real 1769077978] req@ffff9350100c9f80 x1855008913561088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077978.609515:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3684 00000100:00080000:1.0:1769077978.609531:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077978.609537:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077978.609544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077978.609546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913561088/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077978.609561:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077978.609569:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:1.0:1769077978.609570:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913561472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077978.609574:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913561088:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769077978.609600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077978.609604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077978.609607:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077978.609615:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3684 00000100:00100000:0.0:1769077978.609618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913561216:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769077978.609627:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077978.609637:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769077978.609646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077978.609654:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769077978.609692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913561600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077978.609698:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077978/real 1769077978] req@ffff9350100c8000 x1855008913561216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077978.609749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077978.609756:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913561216/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077978.609756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077978.609764:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077978.609780:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3684 00000100:00080000:0.0:1769077978.609788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077978.609788:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077978.609803:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077978.609805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077978.609816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913561216:192.168.204.121@tcp:38:kworker.0 00000008:00100000:1.0:1769077978.609816:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077978.609833:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913561344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077978.609843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913561728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077978.609856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077978.609861:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769077978.609870:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077978.609873:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077978.609877:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077978/real 1769077978] req@ffff9350100c8a80 x1855008913561344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077978.609885:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3684 00000100:00080000:1.0:1769077978.609896:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077978.609897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913561344/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077978.609914:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769077978.609916:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077978.609928:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769077978.609930:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077978.609934:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913561344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077978.609947:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913561472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077978.609957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913561856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077978.609968:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077978.609982:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077978/real 1769077978] req@ffff9350100c9180 x1855008913561472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077978.609996:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913561472/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077978.610007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077978.610010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077978.610013:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913561472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077978.610032:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913561600:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769077978.610043:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077978.610071:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077978/real 1769077978] req@ffff9350100cb100 x1855008913561600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077978.610079:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913561600/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077978.610085:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077978.610087:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077978.610089:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913561600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077978.610093:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913561728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077978.610112:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077978.610123:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077978/real 1769077978] req@ffff9350100c9c00 x1855008913561728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077978.610137:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913561728/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077978.610146:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077978.610150:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077978.610154:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913561728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077978.610161:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913561856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769077978.610177:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769077978.610186:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077978/real 1769077978] req@ffff9350100cad80 x1855008913561856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077978.610194:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913561856/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077978.610199:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077978.610201:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769077978.610203:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913561856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077978.801186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077979.313206:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077979.313214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077979.313215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077979.313286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077979.633166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077979.825153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077980.337165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077980.337174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077980.337210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077980.337225:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077980.657235:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077980.849143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077981.361188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077981.361189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077981.361189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077981.361260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077981.681194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077981.873158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077982.385209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077982.385210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077982.385210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077982.385273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077982.705182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077982.897191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077983.409189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077983.409189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077983.409205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077983.409229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077983.729168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077983.729168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769077983.729177:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077983.729180:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077983.729187:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3689 00000100:00080000:0.0:1769077983.729203:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077983.729208:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769077983.729237:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913561984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077983.729255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077983.729258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077983.729259:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077983.729262:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3689 00000100:00080000:0.0:1769077983.729264:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077983.729267:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077983.729274:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077983.729284:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913562112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077983.729284:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913561984:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769077983.729297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077983.729304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077983.729306:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077983.729310:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3689 00000100:00080000:0.0:1769077983.729314:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077983.729318:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077983.729323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077983.729336:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913562240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077983.729345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077983.729349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077983.729351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077983.729355:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3689 00000100:00080000:0.0:1769077983.729359:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077983.729363:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077983.729368:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769077983.729379:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913562368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077983.729385:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913562112:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769077983.729391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077983.729395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077983.729396:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077983.729400:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3689 00000100:00080000:0.0:1769077983.729404:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077983.729407:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077983.729411:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077983.729411:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913562240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769077983.729425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913562496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077983.729433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077983.729436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077983.729437:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913562368:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769077983.729439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077983.729446:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3689 00000100:00080000:0.0:1769077983.729449:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077983.729452:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077983.729456:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077983.729462:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913562496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769077983.729467:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913562624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077983.729478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077983.729480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077983.729481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077983.729483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3689 00000100:00080000:0.0:1769077983.729485:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077983.729487:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769077983.729489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077983.729497:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913562752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077983.729518:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913562624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077983.729543:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913562752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077983.921209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077984.433150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077984.433150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077984.433183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077984.433542:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077984.753366:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077984.945177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077985.457191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077985.457191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077985.457220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077985.457231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077985.777156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077985.969412:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077986.481175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077986.481176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077986.481202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077986.481360:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077986.801149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769077986.801349:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769077986.801391:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077986.801409:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077986.801447:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077986.801457:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077986.801466:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077986.801476:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769077986.801494:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077986.801506:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077983/real 1769077986] req@ffff9350103fad80 x1855008913562368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769077986.801519:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769077986.801555:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913562368/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077986.801571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077986.801576:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077986.801580:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913562368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077986.801588:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077983/real 1769077986] req@ffff9350103fb480 x1855008913562496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077986.801600:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913562496/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077986.801607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077986.801610:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077986.801613:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913562496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077986.801618:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077983/real 1769077986] req@ffff9350103fb800 x1855008913562112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077986.801627:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913562112/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077986.801635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077986.801638:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077986.801642:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913562112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077986.801648:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077983/real 1769077986] req@ffff9350103f8700 x1855008913562240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077986.801657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913562240/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077986.801664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077986.801668:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077986.801670:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913562240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077986.801677:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077983/real 1769077986] req@ffff9350103faa00 x1855008913561984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077986.801689:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913561984/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077986.801696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077986.801700:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077986.801703:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913561984:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077986.801729:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077983/real 1769077986] req@ffff9350103fb100 x1855008913562624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077986.801741:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913562624/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077986.801748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077986.801751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077986.801754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913562624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077986.801760:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077983/real 1769077986] req@ffff9350103f8380 x1855008913562752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077986.801771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913562752/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077986.801779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077986.801782:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077986.801785:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913562752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077986.993204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077987.505194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077987.505194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077987.505194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077987.505235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077987.825152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077988.017216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077988.529200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077988.529200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077988.529207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077988.529261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077988.849175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077988.849222:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077988.849227:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077988.849231:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077988.849239:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3694 00000100:00080000:3.0:1769077988.849249:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077988.849255:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077988.849293:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913562880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077988.849322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077988.849325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077988.849327:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077988.849331:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3694 00000100:00080000:3.0:1769077988.849335:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077988.849338:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077988.849350:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077988.849365:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913563008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077988.849381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077988.849385:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077988.849387:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077988.849395:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3694 00000100:00100000:2.0:1769077988.849398:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913562880:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077988.849401:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077988.849409:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077988.849427:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077988.849446:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913563136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077988.849454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077988.849458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077988.849459:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077988.849463:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3694 00000100:00080000:3.0:1769077988.849499:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077988.849505:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077988.849513:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077988.849514:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077988.849531:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913563264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077988.849541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077988.849544:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077988.849546:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077988.849551:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3694 00000100:00100000:2.0:1769077988.849577:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077988/real 1769077988] req@ffff935002a7ca80 x1855008913562880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077988.849598:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913562880/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077988.849608:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077988.849613:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077988.849617:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913562880:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769077988.849642:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563008:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077988.849667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077988.849680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077988/real 1769077988] req@ffff935002a7c700 x1855008913563008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077988.849691:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913563008/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077988.849700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077988.849704:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077988.849708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077988.849715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077988.849736:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077988.849745:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077988/real 1769077988] req@ffff935002a7c380 x1855008913563136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077988.849757:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913563136/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077988.849765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077988.849768:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077988.849772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077988.849778:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077988.849794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077988.849804:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077988/real 1769077988] req@ffff935002a7f100 x1855008913563264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077988.849815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913563264/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077988.849823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077988.849826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077988.849829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563264:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077988.849861:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077988.849870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077988.849876:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077988.849893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913563392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077988.850017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077988.850021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077988.850022:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077988.850026:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3694 00000100:00080000:3.0:1769077988.850030:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077988.850033:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077988.850038:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077988.850076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913563520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077988.850085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077988.850088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077988.850089:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077988.850092:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3694 00000100:00080000:3.0:1769077988.850095:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077988.850098:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077988.850102:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077988.850111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913563648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077988.851702:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913563392:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077988.851758:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077988.851780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077988/real 1769077988] req@ffff935002a7f100 x1855008913563392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077988.851798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913563392/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077988.851808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077988.851812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077988.851816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913563392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077988.851823:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913563520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077988.851841:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077988.851850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077988/real 1769077988] req@ffff935002a7c380 x1855008913563520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077988.851861:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913563520/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077988.851868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077988.851872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077988.851874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913563520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077988.851880:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913563648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077988.851894:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077988.851902:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077988/real 1769077988] req@ffff935002a7c700 x1855008913563648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077988.851912:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913563648/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077988.851923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077988.851926:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077988.851929:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913563648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077989.041130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077989.553158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077989.553176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077989.553177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077989.553209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077989.873187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077990.065193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077990.577182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077990.577186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077990.577227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077990.577230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077990.897196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077991.089175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077991.601169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077991.601169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077991.601169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077991.601201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077991.921199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077992.113174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077992.625176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077992.625176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077992.625213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077992.625229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077992.945168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077993.137410:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077993.649175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077993.649175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077993.649442:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077993.649600:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769077993.969140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077993.969147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077993.969152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077993.969158:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077993.969167:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3699 00000100:00080000:3.0:1769077993.969178:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077993.969185:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077993.969222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913563776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077993.969251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077993.969255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077993.969257:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077993.969261:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3699 00000100:00080000:3.0:1769077993.969265:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077993.969269:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077993.969279:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077993.969292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913563904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077993.969295:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913563776:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077993.969309:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077993.969313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077993.969316:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077993.969321:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3699 00000100:00080000:3.0:1769077993.969325:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077993.969329:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077993.969335:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077993.969347:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913564032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077993.969355:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077993.969356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077993.969361:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077993.969362:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077993.969366:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3699 00000100:00080000:3.0:1769077993.969376:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077993.969381:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077993.969386:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077993.969398:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff935002a7ca80 x1855008913563776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077993.969428:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913563776/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077993.969430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913564160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077993.969441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077993.969448:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769077993.969448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077993.969452:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913563776:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077993.969454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077993.969458:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077993.969463:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3699 00000100:00080000:3.0:1769077993.969467:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077993.969471:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769077993.969473:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563904:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769077993.969478:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077993.969492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913564288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077993.969503:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077993.969511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077993.969515:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff935002a7e300 x1855008913563904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077993.969516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077993.969520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077993.969529:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3699 00000100:00100000:2.0:1769077993.969533:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913563904/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077993.969533:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077993.969541:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769077993.969549:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769077993.969549:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769077993.969557:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077993.969562:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913563904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769077993.969571:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913564416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077993.969572:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564032:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077993.969585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077993.969588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077993.969591:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077993.969595:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3699 00000100:00080000:3.0:1769077993.969598:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077993.969603:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077993.969606:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077993.969612:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077993.969622:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff935002a7d180 x1855008913564032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077993.969626:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913564544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077993.969649:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913564032/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077993.969665:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077993.969669:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077993.969671:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077993.969676:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077993.969691:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077993.969697:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff935002a7dc00 x1855008913564160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077993.969704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913564160/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077993.969709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077993.969712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077993.969715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077993.969731:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913564288:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077993.969749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077993.969760:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff935002a7c000 x1855008913564288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077993.969771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913564288/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077993.969776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077993.969778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077993.969780:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913564288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077993.969784:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913564416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077993.969795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077993.969801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff935002a7f800 x1855008913564416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077993.969808:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913564416/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077993.969812:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077993.969814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077993.969816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913564416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077993.969818:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913564544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077993.969826:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077993.969830:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff935002a7df80 x1855008913564544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077993.969836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913564544/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077993.969845:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077993.969846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077993.969848:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913564544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077994.161183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077994.673144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077994.675509:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769077994.675608:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077994.675616:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077994.993189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077995.185105:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769077995.697158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077995.697195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077995.697272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077995.697296:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077996.017211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077996.209189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077996.721185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077996.721185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077996.721236:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077996.721249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077997.041166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077997.233135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077997.745159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077997.745159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077997.745159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769077997.745200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077998.065133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769077998.257178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769077998.769162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077998.769162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077998.769178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077998.769193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077999.089159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077999.089161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769077999.089171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077999.089176:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077999.089186:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3704 00000100:00080000:3.0:1769077999.089197:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077999.089205:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769077999.089244:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913564672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077999.089265:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077999.089269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077999.089271:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077999.089278:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3704 00000100:00080000:3.0:1769077999.089283:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077999.089288:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913564672:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077999.089295:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077999.089308:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077999.089338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913564800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077999.089348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077999.089351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077999.089353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077999.089359:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3704 00000100:00080000:3.0:1769077999.089373:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077999.089378:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769077999.089384:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769077999.089384:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769077999.089405:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913564928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077999.089417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077999.089421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077999.089423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077999.089428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3704 00000100:00080000:3.0:1769077999.089432:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077999.089433:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077999/real 1769077999] req@ffff935002a7fb80 x1855008913564672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077999.089444:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077999.089454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769077999.089458:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913564672/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077999.089472:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077999.089480:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769077999.089482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913565056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077999.089485:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913564672:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769077999.089505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077999.089509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077999.089511:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077999.089515:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3704 00000100:00080000:3.0:1769077999.089519:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077999.089521:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564800:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769077999.089530:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077999.089536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077999.089547:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769077999.089552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913565184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077999.089561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077999.089563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077999.089564:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077999.089569:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077999/real 1769077999] req@ffff935002a7df80 x1855008913564800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077999.089569:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3704 00000100:00080000:3.0:1769077999.089581:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077999.089594:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077999.089607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077999.089611:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913564800/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077999.089622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077999.089624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913565312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077999.089627:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077999.089633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077999.089649:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564928:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769077999.089668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077999.089673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077999.089676:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077999.089680:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769077999.089683:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3704 00000100:00080000:3.0:1769077999.089690:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077999.089695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077999/real 1769077999] req@ffff935002a7f800 x1855008913564928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077999.089695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769077999.089703:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077999.089724:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913564928/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077999.089724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913565440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077999.089786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077999.089791:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077999.089794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913564928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077999.089800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077999.089816:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077999.089823:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077999/real 1769077999] req@ffff935002a7c000 x1855008913565056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077999.089829:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913565056/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077999.089834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077999.089837:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077999.089839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077999.089851:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913565184:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769077999.089860:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077999.089865:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077999/real 1769077999] req@ffff935002a7dc00 x1855008913565184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077999.089872:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913565184/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077999.089883:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077999.089885:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077999.089887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913565184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769077999.089890:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913565312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077999.089900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077999.089906:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077999/real 1769077999] req@ffff935002a7d180 x1855008913565312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077999.089912:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913565312/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077999.089917:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077999.089919:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077999.089920:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913565312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769077999.089924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913565440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769077999.089933:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769077999.089938:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077999/real 1769077999] req@ffff935002a7e300 x1855008913565440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077999.089944:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913565440/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077999.089953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077999.089955:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769077999.089956:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913565440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769077999.281106:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769077999.793135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077999.793161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077999.793161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769077999.793211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078000.113186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078000.305178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078000.817187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078000.817187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078000.817214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078000.817325:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078001.137185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078001.329134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078001.841188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078001.841188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078001.841188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078001.841249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078002.161185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078002.353136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078002.865203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078002.865203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078002.865203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078002.865290:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078003.185203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078003.377129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078003.889179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078003.889179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078003.889180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078003.889334:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078004.209189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078004.209199:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078004.209206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078004.209210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078004.209220:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3709 00000100:00080000:3.0:1769078004.209244:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078004.209253:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078004.209299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913565568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078004.209330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078004.209334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078004.209340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078004.209344:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3709 00000100:00080000:3.0:1769078004.209347:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078004.209350:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078004.209359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078004.209370:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913565696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078004.209378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078004.209380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078004.209382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078004.209385:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3709 00000100:00080000:3.0:1769078004.209387:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078004.209389:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078004.209393:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078004.209397:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913565568:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078004.209403:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913565824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078004.209413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078004.209415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078004.209416:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078004.209419:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3709 00000100:00080000:3.0:1769078004.209421:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078004.209423:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078004.209427:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078004.209434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913565952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078004.209441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078004.209444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078004.209445:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078004.209448:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3709 00000100:00080000:3.0:1769078004.209478:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078004.209482:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078004.209487:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078004.209493:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078004.209505:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913566080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078004.209516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078004.209521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078004.209523:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078004.209528:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3709 00000100:00080000:3.0:1769078004.209532:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078004.209545:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078004/real 1769078004] req@ffff935002a7ca80 x1855008913565568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078004.209546:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078004.209554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078004.209567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913566208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078004.209575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078004.209577:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913565568/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078004.209587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078004.209591:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078004.209597:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3709 00000100:00080000:3.0:1769078004.209600:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078004.209602:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078004.209603:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078004.209610:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078004.209614:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078004.209619:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913566336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078004.209633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913565568:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078004.209645:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565696:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078004.209671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078004.209684:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078004/real 1769078004] req@ffff935002a7c700 x1855008913565696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078004.209693:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913565696/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078004.209700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078004.209702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078004.209705:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565696:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078004.209726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078004.209739:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078004.209746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078004/real 1769078004] req@ffff935002a7c380 x1855008913565824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078004.209754:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913565824/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078004.209760:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078004.209762:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078004.209764:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078004.209767:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078004.209776:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078004.209782:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078004/real 1769078004] req@ffff935002a7f100 x1855008913565952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078004.209788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913565952/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078004.209794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078004.209796:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078004.209798:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913565952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078004.209801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566080:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078004.209810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078004.209823:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078004/real 1769078004] req@ffff935002a7e300 x1855008913566080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078004.209833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913566080/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078004.209839:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078004.209841:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078004.209843:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078004.209847:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078004.209858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078004.209864:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078004/real 1769078004] req@ffff935002a7d180 x1855008913566208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078004.209871:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913566208/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078004.209877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078004.209879:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078004.209881:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078004.209884:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078004.209893:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078004.209899:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078004/real 1769078004] req@ffff935002a7dc00 x1855008913566336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078004.209906:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913566336/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078004.209911:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078004.209913:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078004.209915:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078004.401152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078004.913190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078004.913190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078004.913194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078004.913264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078005.233138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078005.425116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078005.937178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078005.937184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078005.937189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078005.937232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078006.257169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078006.449134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078006.961136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078006.961137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078006.961174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078006.961211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078007.281126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078007.473166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078007.985133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078007.985133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078007.985134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078007.985206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078008.305126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078008.497127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078009.009209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078009.009250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078009.009262:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078009.009287:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078009.329132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078009.329132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078009.329155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078009.329160:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078009.329170:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3715 00000100:00080000:3.0:1769078009.329181:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078009.329188:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078009.329229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913566464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078009.329254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078009.329257:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078009.329259:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078009.329261:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3715 00000100:00080000:3.0:1769078009.329264:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078009.329266:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078009.329275:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078009.329286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913566592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078009.329294:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078009.329297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078009.329298:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078009.329301:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3715 00000100:00080000:3.0:1769078009.329303:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078009.329305:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078009.329309:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078009.329315:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913566720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078009.329324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078009.329326:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078009.329328:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078009.329332:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3715 00000100:00080000:3.0:1769078009.329338:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078009.329339:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913566464:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078009.329343:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078009.329355:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078009.329364:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913566848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078009.329372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078009.329375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078009.329376:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078009.329380:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3715 00000100:00080000:3.0:1769078009.329382:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078009.329386:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078009.329389:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078009.329401:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913566976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078009.329410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078009.329415:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769078009.329419:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078009.329421:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078009.329429:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3715 00000100:00080000:3.0:1769078009.329436:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078009.329444:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078009.329451:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078009.329467:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913567104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078009.329468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078009/real 1769078009] req@ffff935002a7c000 x1855008913566464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078009.329539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078009.329541:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078009.329544:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078009.329551:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913566464/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078009.329552:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3715 00000100:00080000:3.0:1769078009.329559:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078009.329568:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078009.329577:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078009.329586:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913567232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078009.329590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078009.329608:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078009.329615:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913566464:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078009.329625:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913566592:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078009.329645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078009.329654:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078009/real 1769078009] req@ffff935002a7f800 x1855008913566592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078009.329666:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913566592/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078009.329672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078009.329674:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078009.329676:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913566592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078009.329696:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913566720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078009.329707:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078009.329713:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078009/real 1769078009] req@ffff935002a7df80 x1855008913566720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078009.329720:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913566720/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078009.329724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078009.329726:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078009.329728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913566720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078009.329732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913566848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078009.329739:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078009.329744:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078009/real 1769078009] req@ffff935002a7fb80 x1855008913566848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078009.329750:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913566848/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078009.329762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078009.329766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078009.329769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913566848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078009.329774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566976:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078009.329793:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078009.329803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078009/real 1769078009] req@ffff935002a7dc00 x1855008913566976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078009.329814:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913566976/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078009.329822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078009.329825:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078009.329828:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913566976:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078009.329834:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913567104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078009.329853:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078009.329863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078009/real 1769078009] req@ffff935002a7d180 x1855008913567104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078009.329873:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913567104/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078009.329880:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078009.329883:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078009.329887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913567104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078009.329893:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913567232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078009.329911:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078009.329922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078009/real 1769078009] req@ffff935002a7e300 x1855008913567232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078009.329932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913567232/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078009.329937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078009.329939:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078009.329941:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913567232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078009.521198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078010.033214:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078010.033219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078010.033241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078010.033321:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078010.353220:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078010.545181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078011.057173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078011.057182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078011.057216:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078011.057231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078011.377142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078011.569158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078012.081147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078012.081151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078012.081183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078012.081226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078012.401168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078012.593166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078013.105203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078013.105205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078013.105264:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078013.105381:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078013.425262:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078013.617162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078014.129180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078014.129180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078014.129218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078014.129252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078014.449147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078014.449171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078014.449185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078014.449191:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078014.449203:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3720 00000100:00080000:3.0:1769078014.449218:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078014.449227:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078014.449276:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913567360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078014.449312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078014.449316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078014.449319:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078014.449325:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3720 00000100:00080000:3.0:1769078014.449329:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078014.449333:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078014.449346:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078014.449362:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913567488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078014.449374:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913567360:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078014.449380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078014.449385:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078014.449388:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078014.449394:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3720 00000100:00080000:3.0:1769078014.449399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078014.449404:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078014.449410:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078014.449425:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913567616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078014.449435:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078014.449439:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078014.449442:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078014.449446:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3720 00000100:00080000:3.0:1769078014.449450:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078014.449458:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078014.449462:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769078014.449469:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078014.449486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913567744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078014.449497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078014.449502:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078014.449505:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078014.449511:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3720 00000100:00100000:2.0:1769078014.449518:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078014/real 1769078014] req@ffff935002a7f100 x1855008913567360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078014.449520:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078014.449530:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078014.449536:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078014.449554:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913567872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078014.449562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913567360/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078014.449581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078014.449590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078014.449592:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078014.449594:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078014.449603:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078014.449604:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3720 00000100:00080000:3.0:1769078014.449616:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078014.449619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913567360:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078014.449627:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078014.449636:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078014.449650:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913568000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078014.449659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913567488:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078014.449672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078014.449677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078014.449680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078014.449686:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3720 00000100:00080000:3.0:1769078014.449692:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078014.449699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078014.449705:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078014.449713:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078014.449734:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078014/real 1769078014] req@ffff935002a7c380 x1855008913567488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078014.449744:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913568128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078014.449772:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913567488/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078014.449786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078014.449791:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078014.449799:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913567488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078014.449819:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913567616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078014.449842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078014.449855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078014/real 1769078014] req@ffff935002a7c700 x1855008913567616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078014.449869:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913567616/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078014.449878:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078014.449882:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078014.449886:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913567616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078014.449893:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913567744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078014.449912:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078014.449924:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078014/real 1769078014] req@ffff935002a7ca80 x1855008913567744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078014.449938:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913567744/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078014.449947:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078014.449951:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078014.449955:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913567744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078014.449961:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913567872:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078014.450000:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078014.450012:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078014/real 1769078014] req@ffff935002a7e300 x1855008913567872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078014.450032:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913567872/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078014.450041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078014.450045:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078014.450093:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913567872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078014.450109:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078014.450144:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078014.450157:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078014/real 1769078014] req@ffff935002a7d180 x1855008913568000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078014.450170:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913568000/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078014.450185:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078014.450189:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078014.450192:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078014.450199:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078014.450216:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078014.450224:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078014/real 1769078014] req@ffff935002a7dc00 x1855008913568128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078014.450232:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913568128/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078014.450238:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078014.450240:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078014.450242:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078014.641207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078015.153196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078015.153206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078015.153223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078015.153248:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078015.473166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078015.665179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078016.177168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078016.177187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078016.177189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078016.177244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078016.497127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078016.689196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078017.201213:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078017.201213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078017.201215:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078017.201268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078017.521173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078017.713180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078018.225200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078018.225200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078018.225202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078018.225259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078018.545150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078018.737186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078019.249179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078019.249179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078019.249181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078019.249185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078019.569200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078019.569213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078019.569231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078019.569236:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078019.569250:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3725 00000100:00080000:3.0:1769078019.569264:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078019.569272:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078019.569322:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913568256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078019.569346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078019.569350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078019.569351:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078019.569355:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3725 00000100:00080000:3.0:1769078019.569358:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078019.569361:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078019.569371:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078019.569382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913568384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078019.569392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078019.569398:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913568256:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078019.569399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078019.569405:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078019.569410:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3725 00000100:00080000:3.0:1769078019.569412:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078019.569423:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078019.569427:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078019.569439:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913568512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078019.569445:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078019.569448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078019.569450:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078019.569454:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3725 00000100:00080000:3.0:1769078019.569476:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078019.569481:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078019.569486:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769078019.569496:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078019.569516:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913568640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078019.569530:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078019.569533:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078019/real 1769078019] req@ffff935002a7fb80 x1855008913568256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078019.569535:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078019.569539:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078019.569555:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3725 00000100:00080000:3.0:1769078019.569562:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078019.569567:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078019.569571:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913568256/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078019.569575:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078019.569595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078019.569600:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913568768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078019.569609:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078019.569612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078019.569616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078019.569621:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078019.569622:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913568256:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078019.569629:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3725 00000100:00080000:3.0:1769078019.569635:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078019.569643:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078019.569646:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078019.569655:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913568896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078019.569660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078019.569664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078019.569666:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078019.569675:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3725 00000100:00100000:2.0:1769078019.569676:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913568384:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078019.569683:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078019.569691:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078019.569695:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078019.569706:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913569024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078019.569718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078019.569749:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078019/real 1769078019] req@ffff935002a7df80 x1855008913568384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078019.569766:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913568384/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078019.569776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078019.569780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078019.569783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913568384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078019.569789:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913568512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078019.569807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078019.569814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078019/real 1769078019] req@ffff935002a7f800 x1855008913568512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078019.569822:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913568512/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078019.569828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078019.569830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078019.569832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913568512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078019.569837:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913568640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078019.569847:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078019.569853:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078019/real 1769078019] req@ffff935002a7c000 x1855008913568640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078019.569860:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913568640/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078019.569865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078019.569867:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078019.569869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913568640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078019.569873:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568768:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078019.569882:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078019.569887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078019/real 1769078019] req@ffff935002a7dc00 x1855008913568768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078019.569894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913568768/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078019.569900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078019.569902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078019.569903:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078019.569907:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078019.569917:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078019.569922:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078019/real 1769078019] req@ffff935002a7d180 x1855008913568896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078019.569930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913568896/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078019.569935:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078019.569936:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078019.569938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913568896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078019.569953:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078019.569963:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078019.569991:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078019/real 1769078019] req@ffff935002a7e300 x1855008913569024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078019.570001:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913569024/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078019.570006:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078019.570008:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078019.570010:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078019.761171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078020.273158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078020.273158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078020.273158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078020.273186:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078020.593146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078020.785129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078021.297118:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078021.297123:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078021.297140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078021.297155:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078021.617157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078021.809163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078022.321141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078022.321141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078022.321147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078022.321177:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078022.641157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078022.833151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078023.345157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078023.345157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078023.345157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078023.345208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078023.665212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078023.857168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078024.369171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078024.369171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078024.369171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078024.369214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078024.689179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078024.689212:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078024.689224:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078024.689230:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078024.689244:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3730 00000100:00080000:1.0:1769078024.689257:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078024.689266:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078024.689316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913569152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078024.689345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078024.689348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078024.689349:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078024.689352:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3730 00000100:00080000:1.0:1769078024.689356:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078024.689358:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078024.689370:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078024.689392:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913569280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078024.689404:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913569152:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078024.689410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078024.689415:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078024.689417:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078024.689423:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3730 00000100:00080000:1.0:1769078024.689429:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078024.689435:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078024.689441:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078024.689455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913569408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078024.689465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078024.689468:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078024.689470:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078024.689473:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3730 00000100:00080000:1.0:1769078024.689477:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078024.689481:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078024.689484:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078024.689498:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913569536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078024.689503:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078024.689511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078024.689515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078024.689516:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078024.689520:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3730 00000100:00080000:1.0:1769078024.689525:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078024.689530:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078024.689535:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078024.689543:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913569664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078024.689566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078024.689570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078024.689579:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078024.689580:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078024/real 1769078024] req@ffff9350100c9880 x1855008913569152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078024.689592:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3730 00000100:00080000:1.0:1769078024.689603:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078024.689614:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078024.689620:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078024.689641:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913569792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078024.689652:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913569152/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078024.689659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078024.689677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078024.689681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078024.689683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078024.689693:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3730 00000100:00080000:2.0:1769078024.689696:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078024.689704:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078024.689709:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913569152:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078024.689714:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078024.689726:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078024.689743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913569920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078024.689751:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913569280:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078024.689790:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078024.689807:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078024/real 1769078024] req@ffff9350100c9500 x1855008913569280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078024.689821:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913569280/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078024.689831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078024.689834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078024.689838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913569280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078024.689846:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913569408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078024.689867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078024.689879:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078024/real 1769078024] req@ffff9350100caa00 x1855008913569408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078024.689909:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913569408/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078024.689923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078024.689927:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078024.689931:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913569408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078024.689938:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913569536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078024.689960:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078024.689971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078024/real 1769078024] req@ffff9350100c8380 x1855008913569536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078024.689982:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913569536/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078024.689990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078024.689994:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078024.689997:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913569536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078024.690004:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569664:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078024.690035:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078024.690047:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078024/real 1769078024] req@ffff9350100ca680 x1855008913569664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078024.690088:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913569664/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078024.690097:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078024.690101:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078024.690105:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078024.690110:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078024.690126:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078024.690134:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078024/real 1769078024] req@ffff9350100c8e00 x1855008913569792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078024.690142:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913569792/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078024.690147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078024.690149:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078024.690152:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078024.690165:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078024.690175:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078024.690181:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078024/real 1769078024] req@ffff9350100ca300 x1855008913569920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078024.690189:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913569920/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078024.690194:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078024.690196:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078024.690198:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913569920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078024.881214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078025.393156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078025.393160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078025.393205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078025.393270:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078025.713178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078025.905136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078026.417159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078026.417159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078026.417159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078026.417191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078026.737170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078026.929163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078027.441164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078027.441164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078027.441196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078027.441283:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078027.761179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078027.953140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078028.465180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078028.465193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078028.465194:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078028.465248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078028.785130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078028.977175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078029.489168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078029.489189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078029.489190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078029.489191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:0.0:1769078029.809197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078029.809206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078029.809209:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078029.809217:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078029.809231:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3735 00000100:00080000:0.0:1769078029.809244:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078029.809254:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078029.809298:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913570048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078029.809333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078029.809337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078029.809340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078029.809344:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3735 00000100:00080000:0.0:1769078029.809347:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078029.809352:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078029.809363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078029.809377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913570176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078029.809384:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913570048:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078029.809395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078029.809401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078029.809402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078029.809408:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3735 00000100:00080000:0.0:1769078029.809411:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078029.809415:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078029.809420:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078029.809432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913570304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078029.809442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078029.809446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078029.809447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078029.809451:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3735 00000100:00080000:0.0:1769078029.809456:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078029.809460:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078029.809465:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078029.809468:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078029.809481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913570432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078029.809493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078029.809497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078029.809498:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078029.809503:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3735 00000100:00080000:0.0:1769078029.809507:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078029.809512:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078029.809517:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078029.809521:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078029/real 1769078029] req@ffff9350103f9180 x1855008913570048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078029.809531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913570560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078029.809545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078029.809551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769078029.809552:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913570048/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078029.809554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078029.809563:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3735 00000100:00080000:0.0:1769078029.809571:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078029.809574:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078029.809578:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078029.809582:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:0.0:1769078029.809587:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078029.809588:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913570048:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078029.809606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913570688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078029.809616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078029.809620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078029.809622:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078029.809626:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913570176:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078029.809631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3735 00000100:00080000:0.0:1769078029.809636:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078029.809640:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078029.809647:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078029.809653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078029.809661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913570816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078029.809667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078029/real 1769078029] req@ffff9350103f9500 x1855008913570176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078029.809687:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913570176/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078029.809695:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078029.809699:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078029.809702:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913570176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078029.809707:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913570304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078029.809720:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078029.809727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078029/real 1769078029] req@ffff9350103fbb80 x1855008913570304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078029.809736:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913570304/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078029.809741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078029.809744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078029.809746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913570304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078029.809749:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913570432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078029.809758:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078029.809764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078029/real 1769078029] req@ffff9350103f9c00 x1855008913570432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078029.809772:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913570432/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078029.809778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078029.809780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078029.809781:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913570432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078029.809785:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913570560:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078029.809794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078029.809816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078029/real 1769078029] req@ffff9350103f9f80 x1855008913570560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078029.809825:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913570560/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078029.809831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078029.809834:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078029.809839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913570560:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078029.809843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913570688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078029.809856:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078029.809862:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078029/real 1769078029] req@ffff9350103f8380 x1855008913570688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078029.809870:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913570688/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078029.809876:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078029.809878:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078029.809880:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913570688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078029.809895:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913570816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078029.809905:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078029.809910:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078029/real 1769078029] req@ffff9350103fb100 x1855008913570816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078029.809917:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913570816/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078029.809923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078029.809925:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078029.809928:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913570816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078030.001128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078030.513179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078030.513179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078030.513179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078030.513218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078030.833188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078031.025205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078031.537175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078031.537175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078031.537176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078031.537235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078031.857127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078032.049126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078032.561208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078032.561209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078032.561209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078032.561249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078032.881157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078033.073181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078033.585151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078033.585151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078033.585189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078033.585307:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078033.905140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078034.097196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078034.609123:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078034.609126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078034.609133:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078034.609171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078034.929197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078034.929213:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078034.929219:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078034.929233:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3740 00000100:00080000:1.0:1769078034.929247:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078034.929256:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078034.929259:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078034.929317:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913570944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078034.929350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078034.929356:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078034.929358:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078034.929363:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3740 00000100:00080000:1.0:1769078034.929367:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078034.929372:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078034.929385:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078034.929386:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913570944:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078034.929404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913571072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078034.929416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078034.929419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078034.929421:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078034.929426:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3740 00000100:00080000:1.0:1769078034.929430:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078034.929434:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078034.929439:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078034.929451:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913571200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078034.929461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769078034.929464:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078034.929468:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078034.929471:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078034.929476:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3740 00000100:00080000:1.0:1769078034.929480:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078034.929484:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078034.929490:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078034.929502:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913571328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078034.929511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078034.929516:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9350100cb800 x1855008913570944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078034.929517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078034.929521:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078034.929530:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3740 00000100:00080000:1.0:1769078034.929534:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078034.929537:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913570944/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078034.929542:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078034.929548:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769078034.929550:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078034.929553:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078034.929558:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913570944:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078034.929568:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913571456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078034.929580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078034.929581:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571072:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078034.929588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078034.929591:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078034.929596:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3740 00000100:00080000:1.0:1769078034.929601:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078034.929603:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078034.929609:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078034.929616:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078034.929617:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9350100ca300 x1855008913571072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078034.929632:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913571584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078034.929635:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913571072/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078034.929649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078034.929656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078034.929660:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078034.929664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078034.929667:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3740 00000100:00080000:2.0:1769078034.929672:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078034.929674:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078034.929678:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571072:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078034.929682:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078034.929687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571200:192.168.204.121@tcp:8:kworker.0 00000008:00100000:1.0:1769078034.929689:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078034.929704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913571712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078034.929729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078034.929749:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9350100c8e00 x1855008913571200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.929761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913571200/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.929769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.929772:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078034.929783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078034.929792:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078034.929808:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078034.929816:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9350100ca680 x1855008913571328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.929825:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913571328/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.929832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.929835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078034.929839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078034.929865:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913571456:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078034.929884:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078034.929891:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9350100c8380 x1855008913571456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.929901:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913571456/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.929910:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.929912:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078034.929916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913571456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078034.929921:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913571584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078034.929938:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078034.929948:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9350100caa00 x1855008913571584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.929959:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913571584/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.929967:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.929970:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078034.929973:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913571584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078034.929979:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913571712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078034.929995:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078034.930005:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9350100c9500 x1855008913571712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.930017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913571712/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.930029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.930032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078034.930036:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913571712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078035.121210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078035.633175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078035.633175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078035.633175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078035.633205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078035.953139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078036.145194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078036.657160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078036.657160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078036.657161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078036.657204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078036.977168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078037.169176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078037.681197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078037.681198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078037.681198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078037.681255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078038.001108:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078038.193140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078038.705139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078038.705140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078038.705182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078038.705223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078039.025205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078039.217168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078039.729159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078039.729159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078039.729159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078039.729233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078040.049144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078040.049152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078040.049155:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078040.049162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078040.049168:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3745 00000100:00080000:1.0:1769078040.049179:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078040.049184:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078040.049215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913571840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078040.049242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078040.049247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078040.049249:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078040.049252:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3745 00000100:00080000:1.0:1769078040.049256:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078040.049261:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078040.049270:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078040.049283:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913571968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078040.049297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078040.049300:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078040.049303:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078040.049312:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913571840:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078040.049318:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3745 00000100:00080000:1.0:1769078040.049326:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078040.049331:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078040.049337:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078040.049351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913572096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078040.049361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078040.049366:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078040.049368:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078040.049371:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3745 00000100:00080000:1.0:1769078040.049376:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078040.049380:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078040.049386:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078040.049395:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769078040.049401:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913572224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078040.049410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078040.049414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078040.049416:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078040.049420:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3745 00000100:00080000:1.0:1769078040.049426:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078040.049430:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078040.049437:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078040.049442:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078040/real 1769078040] req@ffff9350100c9880 x1855008913571840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078040.049454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913572352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078040.049465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078040.049466:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913571840/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078040.049472:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078040.049474:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078040.049481:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3745 00000100:00080000:1.0:1769078040.049490:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078040.049497:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078040.049502:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078040.049504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769078040.049514:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913572480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078040.049515:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078040.049528:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913571840:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078040.049536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078040.049539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078040.049541:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078040.049545:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3745 00000100:00080000:1.0:1769078040.049547:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078040.049551:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078040.049555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078040.049566:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769078040.049568:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913572608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078040.049594:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078040.049609:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078040/real 1769078040] req@ffff9350100cad80 x1855008913571968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078040.049618:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913571968/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078040.049654:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078040.049657:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078040.049659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913571968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078040.049664:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078040.049681:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078040.049688:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078040/real 1769078040] req@ffff9350100c9c00 x1855008913572096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078040.049695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913572096/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078040.049700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078040.049702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078040.049704:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078040.049708:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078040.049716:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078040.049721:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078040/real 1769078040] req@ffff9350100cb100 x1855008913572224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078040.049728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913572224/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078040.049732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078040.049734:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078040.049736:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078040.049740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913572352:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078040.049749:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078040.049754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078040/real 1769078040] req@ffff9350100c9180 x1855008913572352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078040.049761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913572352/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078040.049765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078040.049767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078040.049769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913572352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078040.049773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913572480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078040.049782:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078040.049786:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078040/real 1769078040] req@ffff9350100c8a80 x1855008913572480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078040.049793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913572480/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078040.049797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078040.049798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078040.049800:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913572480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078040.049812:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913572608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078040.049821:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078040.049827:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078040/real 1769078040] req@ffff9350100c8000 x1855008913572608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078040.049833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913572608/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078040.049838:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078040.049839:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078040.049841:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913572608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078040.241171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078040.753166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078040.753166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078040.753183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078040.753200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078041.073188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078041.265200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078041.777182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078041.777203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078041.777203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078041.777251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078042.097195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078042.289164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078042.801174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078042.801181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078042.801204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078042.801240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078043.121154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078043.313119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078043.825182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078043.825182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078043.825195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078043.825261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078044.145127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078044.337167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078044.849170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078044.849170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078044.849199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078044.849205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078045.169147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078045.169149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078045.169159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078045.169163:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078045.169172:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3750 00000100:00080000:1.0:1769078045.169183:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078045.169189:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078045.169233:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913572736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078045.169255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078045.169259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078045.169260:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078045.169263:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3750 00000100:00080000:1.0:1769078045.169265:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078045.169267:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078045.169275:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078045.169286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913572864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078045.169293:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078045.169295:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078045.169296:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078045.169299:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3750 00000100:00080000:1.0:1769078045.169302:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078045.169310:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913572736:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078045.169311:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078045.169317:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078045.169327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913572992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078045.169337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078045.169340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078045.169342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078045.169346:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3750 00000100:00080000:1.0:1769078045.169350:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078045.169355:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078045.169360:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078045.169371:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913573120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078045.169379:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078045.169381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078045.169382:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078045.169385:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3750 00000100:00080000:1.0:1769078045.169388:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078045.169392:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078045.169395:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078045.169401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078045.169412:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913573248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078045.169422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078045.169425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078045.169427:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078045.169432:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3750 00000100:00080000:1.0:1769078045.169436:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078045.169442:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078045/real 1769078045] req@ffff9350100c9f80 x1855008913572736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078045.169443:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078045.169451:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078045.169466:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913573376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078045.169472:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913572736/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078045.169475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078045.169482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078045.169485:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078045.169491:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3750 00000100:00080000:2.0:1769078045.169494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078045.169495:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078045.169500:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078045.169503:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078045.169504:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078045.169510:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913572736:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078045.169512:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913573504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078045.169522:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078045.169560:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078045.169572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078045/real 1769078045] req@ffff9350100c8000 x1855008913572864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078045.169591:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913572864/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078045.169600:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078045.169604:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078045.169607:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078045.169631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078045.169656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078045.169666:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078045/real 1769078045] req@ffff9350100c8a80 x1855008913572992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078045.169678:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913572992/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078045.169686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078045.169689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078045.169693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913572992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078045.169699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913573120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078045.169714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078045.169723:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078045/real 1769078045] req@ffff9350100c9180 x1855008913573120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078045.169734:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913573120/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078045.169748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078045.169751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078045.169754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913573120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078045.169760:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913573248:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078045.169774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078045.169782:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078045/real 1769078045] req@ffff9350100cb100 x1855008913573248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078045.169793:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913573248/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078045.169801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078045.169804:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078045.169808:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913573248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078045.169813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913573376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078045.169826:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078045.169834:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078045/real 1769078045] req@ffff9350100c9c00 x1855008913573376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078045.169845:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913573376/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078045.169852:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078045.169855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078045.169859:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913573376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078045.169864:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913573504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078045.169878:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078045.169886:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078045/real 1769078045] req@ffff9350100cad80 x1855008913573504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078045.169898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913573504/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078045.169906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078045.169909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078045.169912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913573504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078045.361152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078045.873167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078045.873167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078045.873167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078045.873202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078046.193160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078046.385154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078046.897191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078046.897191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078046.897192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078046.897254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078047.217194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078047.409131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078047.921182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078047.921183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078047.921220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078047.921307:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078048.241147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078048.433185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078048.945137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078048.945183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078048.945193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078048.945277:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078049.265148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078049.457138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078049.969184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078049.969184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078049.969192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078049.969259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078050.289184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078050.289186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078050.289197:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078050.289201:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078050.289211:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3755 00000100:00080000:3.0:1769078050.289221:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078050.289229:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078050.289261:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913573632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078050.289286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078050.289288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078050.289290:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078050.289293:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3755 00000100:00080000:3.0:1769078050.289295:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078050.289297:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078050.289305:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078050.289312:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913573760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078050.289321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078050.289324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078050.289326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078050.289330:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913573632:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078050.289334:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3755 00000100:00080000:3.0:1769078050.289339:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078050.289343:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078050.289349:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078050.289362:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913573888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078050.289371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078050.289374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078050.289376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078050.289380:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3755 00000100:00080000:3.0:1769078050.289383:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078050.289386:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078050.289391:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078050.289401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913574016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078050.289411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078050.289415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1769078050.289419:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913573760:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078050.289419:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078050.289428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3755 00000100:00080000:3.0:1769078050.289433:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078050.289438:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078050.289443:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078050.289470:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913573888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078050.289476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913574144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078050.289488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078050.289493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078050.289494:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078050.289498:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574016:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769078050.289502:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3755 00000100:00080000:3.0:1769078050.289508:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078050.289513:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078050.289519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078050.289529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913574272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078050.289535:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913574144:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078050.289540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078050.289543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078050.289545:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078050.289549:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3755 00000100:00080000:3.0:1769078050.289552:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078050.289554:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078050.289557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078050.289565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913574400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078050.289569:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913574272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078050.289591:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913574400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078050.481153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078050.993125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078050.993151:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078050.993163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078050.993173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078051.313196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078051.505152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078052.017144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078052.017164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078052.017166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078052.017244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078052.337218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078052.529161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078053.041178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078053.041178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078053.041200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078053.041214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078053.361326:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769078053.361461:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769078053.361507:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078053.361529:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078053.361577:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078053.361587:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078053.361595:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078053.361604:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078053.361612:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078053.361621:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078053.361703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078050/real 1769078053] req@ffff935002a7dc00 x1855008913574400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078053.361728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913574400/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078053.361741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078053.361747:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078053.361764:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913574400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078053.361773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078050/real 1769078053] req@ffff935002a7d180 x1855008913574272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078053.361787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913574272/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078053.361795:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078053.361798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078053.361802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913574272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078053.361810:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078050/real 1769078053] req@ffff935002a7e300 x1855008913574144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078053.361822:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913574144/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078053.361831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078053.361836:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078053.361839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913574144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078053.361846:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078050/real 1769078053] req@ffff935002a7c380 x1855008913573888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078053.361857:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913573888/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078053.361866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078053.361869:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078053.361872:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913573888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078053.361880:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078050/real 1769078053] req@ffff935002a7f100 x1855008913574016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078053.361891:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913574016/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078053.361899:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078053.361902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078053.361905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078053.361912:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078050/real 1769078053] req@ffff935002a7ca80 x1855008913573632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078053.361924:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913573632/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078053.361931:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078053.361935:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078053.361938:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913573632:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078053.361945:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078050/real 1769078053] req@ffff935002a7c700 x1855008913573760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078053.361957:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913573760/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078053.361965:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078053.361969:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078053.361972:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913573760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769078053.553172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078054.065167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078054.065199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078054.065200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078054.065257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078054.385243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078054.577164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078055.089199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078055.089204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078055.089228:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078055.089266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078055.409152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078055.409164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078055.409168:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078055.409179:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3761 00000100:00080000:3.0:1769078055.409191:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078055.409199:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078055.409203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078055.409239:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913574528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078055.409271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078055.409274:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078055.409276:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078055.409279:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3761 00000100:00080000:3.0:1769078055.409281:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078055.409284:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078055.409294:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078055.409304:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913574656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078055.409312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078055.409316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078055.409318:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078055.409323:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3761 00000100:00080000:3.0:1769078055.409327:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078055.409331:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078055.409337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078055.409348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913574528:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078055.409352:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913574784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078055.409393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078055.409398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078055.409400:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078055.409404:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3761 00000100:00080000:3.0:1769078055.409409:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078055.409413:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078055.409419:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078055.409433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913574912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078055.409451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078055.409454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078055.409457:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078055.409459:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078055.409465:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3761 00000100:00080000:3.0:1769078055.409470:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078055.409476:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078055.409482:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078055.409500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913575040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078055.409512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078055.409515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078055.409518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078055.409521:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078055/real 1769078055] req@ffff935002a7c000 x1855008913574528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078055.409541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3761 00000100:00080000:3.0:1769078055.409546:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078055.409549:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913574528/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078055.409555:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078055.409565:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078055.409572:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078055.409580:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078055.409583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913575168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078055.409590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913574528:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078055.409602:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574656:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078055.409605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078055.409613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078055.409619:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078055.409624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3761 00000100:00080000:3.0:1769078055.409628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078055.409632:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078055.409638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078055.409646:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078055.409654:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913575296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078055.409667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078055/real 1769078055] req@ffff935002a7f800 x1855008913574656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078055.409686:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913574656/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078055.409697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078055.409701:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078055.409705:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078055.409741:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078055.409764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078055.409776:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078055/real 1769078055] req@ffff935002a7df80 x1855008913574784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078055.409789:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913574784/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078055.409797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078055.409801:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078055.409804:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078055.409810:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078055.409829:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078055.409854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078055/real 1769078055] req@ffff935002a7fb80 x1855008913574912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078055.409874:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913574912/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078055.409882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078055.409885:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078055.409889:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913574912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078055.409895:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575040:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078055.409914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078055.409925:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078055/real 1769078055] req@ffff935002a7c700 x1855008913575040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078055.409937:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913575040/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078055.409945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078055.409949:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078055.409952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078055.409958:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078055.409977:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078055.409988:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078055/real 1769078055] req@ffff935002a7ca80 x1855008913575168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078055.410000:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913575168/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078055.410014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078055.410018:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078055.410021:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078055.410028:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078055.410047:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078055.410107:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078055/real 1769078055] req@ffff935002a7f100 x1855008913575296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078055.410122:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913575296/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078055.410131:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078055.410135:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078055.410138:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078055.601183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078056.113180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078056.113180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078056.113180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078056.113399:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078056.433308:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078056.625196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078057.137199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078057.137199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078057.137229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078057.137256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078057.457229:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078057.649224:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078058.161147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078058.161147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078058.161147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078058.161179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078058.481184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078058.673156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078059.185202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078059.185204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078059.185239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078059.185316:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078059.505194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078059.697160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078060.209134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078060.209148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078060.209200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078060.209224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078060.529188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078060.529188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078060.529201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078060.529206:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078060.529216:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3766 00000100:00080000:1.0:1769078060.529228:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078060.529234:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078060.529276:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913575424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078060.529316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078060.529319:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078060.529320:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078060.529324:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3766 00000100:00080000:1.0:1769078060.529327:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078060.529330:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078060.529341:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078060.529351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913575552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078060.529360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078060.529363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078060.529364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078060.529370:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3766 00000100:00100000:2.0:1769078060.529370:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913575424:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078060.529376:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078060.529380:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078060.529384:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078060.529393:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913575680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078060.529403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078060.529406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078060.529408:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078060.529413:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3766 00000100:00080000:1.0:1769078060.529417:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078060.529421:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078060.529426:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078060.529440:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913575808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078060.529451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078060.529458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1769078060.529458:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078060.529463:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078060.529467:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3766 00000100:00080000:1.0:1769078060.529471:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078060.529475:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078060.529480:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078060.529490:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913575936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078060.529497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078060.529500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078060.529502:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078060.529504:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3766 00000100:00100000:2.0:1769078060.529510:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078060/real 1769078060] req@ffff9350100c9880 x1855008913575424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078060.529514:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078060.529522:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078060.529527:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078060.529535:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913576064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078060.529543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078060.529546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913575424/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078060.529547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078060.529551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078060.529559:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3766 00000100:00080000:1.0:1769078060.529564:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078060.529571:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078060.529576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769078060.529577:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078060.529583:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078060.529589:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913575424:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078060.529596:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913576192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078060.529615:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913575552:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078060.529636:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078060.529645:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078060/real 1769078060] req@ffff9350100c9500 x1855008913575552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078060.529654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913575552/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078060.529661:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078060.529669:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078060.529672:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913575552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078060.529677:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913575680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078060.529690:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078060.529697:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078060/real 1769078060] req@ffff9350100caa00 x1855008913575680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078060.529704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913575680/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078060.529710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078060.529712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078060.529715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913575680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078060.529719:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913575808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078060.529729:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078060.529735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078060/real 1769078060] req@ffff9350100c8380 x1855008913575808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078060.529743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913575808/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078060.529748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078060.529750:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078060.529752:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913575808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078060.529755:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078060.529766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078060.529771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078060/real 1769078060] req@ffff9350100ca680 x1855008913575936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078060.529779:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913575936/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078060.529784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078060.529786:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078060.529788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913575936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078060.529791:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078060.529801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078060.529806:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078060/real 1769078060] req@ffff9350100c8e00 x1855008913576064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078060.529814:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913576064/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078060.529819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078060.529821:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078060.529823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078060.529826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078060.529835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078060.529840:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078060/real 1769078060] req@ffff9350100ca300 x1855008913576192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078060.529847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913576192/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078060.529853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078060.529855:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078060.529857:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078060.721218:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078061.233141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078061.233142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078061.233142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078061.233293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078061.553160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078061.745148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078062.257205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078062.257205:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078062.257205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078062.257239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078062.577197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078062.769183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078063.281185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078063.281186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078063.281186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078063.281228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078063.601160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078063.793185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078064.305162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078064.305162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078064.305162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078064.305191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078064.625198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078064.817164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078065.329137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078065.329137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078065.329138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078065.329196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078065.649159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078065.649180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078065.649190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078065.649208:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078065.649220:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3771 00000100:00080000:3.0:1769078065.649234:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078065.649244:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078065.649295:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913576320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078065.649337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078065.649342:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078065.649344:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078065.649349:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3771 00000100:00080000:3.0:1769078065.649354:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078065.649360:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078065.649372:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078065.649386:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913576320:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078065.649390:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913576448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078065.649406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078065.649410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078065.649412:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078065.649417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3771 00000100:00080000:3.0:1769078065.649422:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078065.649425:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078065.649432:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078065.649445:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913576576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078065.649456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078065.649460:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078065.649462:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078065.649466:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3771 00000100:00080000:3.0:1769078065.649471:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078065.649476:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078065.649480:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078065.649487:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078065.649510:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913576704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078065.649521:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078065.649525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078065.649527:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078065.649534:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078065/real 1769078065] req@ffff935002a7c380 x1855008913576320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078065.649534:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3771 00000100:00080000:3.0:1769078065.649539:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078065.649547:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078065.649555:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078065.649574:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913576832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078065.649580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913576320/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078065.649591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078065.649598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078065.649600:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078065.649601:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078065.649608:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078065.649612:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3771 00000100:00100000:2.0:1769078065.649615:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913576320:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078065.649619:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078065.649626:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078065.649632:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078065.649643:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913576448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078065.649646:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913576960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078065.649660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078065.649664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078065.649666:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078065.649670:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078065.649674:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3771 00000100:00080000:3.0:1769078065.649679:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078065.649683:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078065.649688:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078065/real 1769078065] req@ffff935002a7e300 x1855008913576448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078065.649688:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078065.649702:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913577088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078065.649704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913576448/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078065.649726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078065.649729:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078065.649731:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913576448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078065.649736:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913576576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078065.649750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078065.649756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078065/real 1769078065] req@ffff935002a7d180 x1855008913576576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078065.649764:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913576576/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078065.649774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078065.649776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078065.649779:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913576576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078065.649782:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913576704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078065.649791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078065.649797:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078065/real 1769078065] req@ffff935002a7dc00 x1855008913576704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078065.649804:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913576704/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078065.649810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078065.649811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078065.649813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913576704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078065.649817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078065.649827:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078065.649832:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078065/real 1769078065] req@ffff935002a7f100 x1855008913576832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078065.649840:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913576832/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078065.649845:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078065.649847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078065.649850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078065.649863:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078065.649873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078065.649879:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078065/real 1769078065] req@ffff935002a7ca80 x1855008913576960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078065.649887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913576960/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078065.649892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078065.649894:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078065.649896:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913576960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078065.649899:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078065.649907:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078065.649913:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078065/real 1769078065] req@ffff935002a7c700 x1855008913577088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078065.649919:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913577088/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078065.649925:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078065.649929:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078065.649932:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078065.841230:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078066.353163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078066.353163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078066.353166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078066.353207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078066.673167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078066.865142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078067.377148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078067.377188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078067.377189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078067.377239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078067.697123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078067.889227:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078068.401168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078068.401170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078068.401188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078068.401298:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078068.721186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078068.913244:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078069.425196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078069.425196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078069.425196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078069.425248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078069.745198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078069.937208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078070.449166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078070.449166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078070.449167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078070.449231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078070.769179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078070.769196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078070.769206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078070.769211:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078070.769228:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3776 00000100:00080000:2.0:1769078070.769241:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078070.769248:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078070.769289:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913577216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078070.769315:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078070.769318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078070.769320:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078070.769323:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3776 00000100:00080000:2.0:1769078070.769326:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078070.769329:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078070.769340:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078070.769349:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913577344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078070.769357:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913577216:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078070.769381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078070.769383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078070.769385:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078070.769390:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3776 00000100:00080000:2.0:1769078070.769392:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078070.769395:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078070.769400:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078070.769415:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913577472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078070.769422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078070.769425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078070.769426:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078070.769429:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3776 00000100:00080000:2.0:1769078070.769431:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078070.769433:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078070.769436:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078070.769443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913577600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078070.769448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078070.769451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078070.769452:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769078070.769460:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078070.769460:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3776 00000100:00080000:2.0:1769078070.769468:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078070.769472:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078070.769476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078070.769489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913577728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078070.769496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078070.769500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078070.769502:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078070.769508:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3776 00000100:00080000:2.0:1769078070.769514:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078070.769519:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078070.769526:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078070.769527:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078070/real 1769078070] req@ffff935002a63100 x1855008913577216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078070.769551:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913577856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078070.769553:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913577216/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078070.769618:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078070.769624:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769078070.769631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078070.769632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913577216:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078070.769646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078070.769649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078070.769654:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3776 00000100:00080000:2.0:1769078070.769657:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078070.769660:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078070.769666:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078070.769673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913577344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078070.769682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913577984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078070.769710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078070.769735:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078070/real 1769078070] req@ffff935002a61500 x1855008913577344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078070.769748:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913577344/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078070.769758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078070.769761:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078070.769765:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913577344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078070.769773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913577472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078070.769794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078070.769806:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078070/real 1769078070] req@ffff935002a60000 x1855008913577472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078070.769820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913577472/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078070.769826:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078070.769828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078070.769831:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913577472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078070.769835:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913577600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078070.769849:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078070.769856:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078070/real 1769078070] req@ffff935002a62a00 x1855008913577600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078070.769864:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913577600/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078070.769870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078070.769873:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078070.769875:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913577600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078070.769879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078070.769888:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078070.769894:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078070/real 1769078070] req@ffff935002a61c00 x1855008913577728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078070.769901:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913577728/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078070.769907:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078070.769909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078070.769911:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078070.769915:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078070.769924:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078070.769930:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078070/real 1769078070] req@ffff935002a61f80 x1855008913577856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078070.769937:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913577856/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078070.769942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078070.769944:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078070.769946:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078070.769961:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078070.769971:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078070.769977:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078070/real 1769078070] req@ffff935002a60a80 x1855008913577984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078070.769985:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913577984/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078070.769991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078070.769992:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078070.769994:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913577984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078070.961159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078071.473200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078071.473203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078071.473203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078071.473262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078071.793216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078071.985161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078072.497127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078072.497132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078072.497172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078072.497243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078072.817171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078073.009177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078073.521214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078073.521231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078073.521340:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078073.521379:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078073.841222:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078074.033164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078074.545162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078074.545189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078074.545189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078074.545295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078074.865188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078075.057152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078075.569154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078075.569173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078075.569200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078075.569207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:2.0:1769078075.889189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078075.889200:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078075.889206:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078075.889216:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3781 00000100:00100000:0.0:1769078075.889223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078075.889229:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078075.889238:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078075.889281:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913578112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078075.889312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078075.889316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078075.889317:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078075.889320:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3781 00000100:00080000:2.0:1769078075.889323:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078075.889326:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078075.889338:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078075.889351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913578240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078075.889363:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078075.889368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769078075.889372:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913578112:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078075.889372:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078075.889382:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3781 00000100:00080000:2.0:1769078075.889387:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078075.889393:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078075.889398:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078075.889408:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913578368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078075.889416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078075.889419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078075.889421:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078075.889423:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3781 00000100:00080000:2.0:1769078075.889427:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078075.889430:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078075.889434:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078075.889441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913578496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078075.889447:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078075.889450:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078075.889452:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078075.889454:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3781 00000100:00080000:2.0:1769078075.889456:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078075.889461:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078075.889462:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078075.889468:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078075.889477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913578624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078075.889484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078075.889487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078075.889488:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078075.889491:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3781 00000100:00080000:2.0:1769078075.889493:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078075.889496:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078075.889498:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078075.889505:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913578752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078075.889516:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078075.889518:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078075.889521:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078075.889523:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff935002a62680 x1855008913578112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078075.889526:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3781 00000100:00080000:2.0:1769078075.889530:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078075.889535:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078075.889541:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078075.889552:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913578880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078075.889562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913578112/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078075.889576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078075.889582:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078075.889587:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913578112:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078075.889618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913578240:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078075.889644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078075.889658:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff935002a61180 x1855008913578240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078075.889672:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913578240/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078075.889681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078075.889685:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078075.889689:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913578240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078075.889696:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913578368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078075.889719:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078075.889730:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff935002a61880 x1855008913578368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078075.889738:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913578368/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078075.889749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078075.889751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078075.889754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913578368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078075.889757:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913578496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078075.889770:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078075.889776:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff935002a62300 x1855008913578496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078075.889794:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913578496/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078075.889803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078075.889806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078075.889810:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913578496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078075.889816:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913578624:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078075.889837:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078075.889850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff935002a60a80 x1855008913578624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078075.889863:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913578624/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078075.889871:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078075.889873:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078075.889876:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913578624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078075.889880:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913578752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078075.889896:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078075.889903:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff935002a61f80 x1855008913578752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078075.889911:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913578752/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078075.889916:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078075.889918:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078075.889921:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913578752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078075.889924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913578880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078075.889937:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078075.889944:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff935002a61c00 x1855008913578880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078075.889951:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913578880/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078075.889965:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078075.889967:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078075.889969:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913578880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078076.081193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078076.593200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078076.593200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078076.593200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078076.593240:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078076.913241:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078077.105154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078077.617177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078077.617187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078077.617187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078077.617217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078077.937174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078078.129198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078078.641172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078078.641172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078078.641172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078078.641204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078078.961140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078079.153175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078079.665160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078079.665160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078079.665160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078079.665194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078079.985147:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078080.177165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078080.689189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078080.689215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078080.689224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078080.689238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078081.009141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769078081.009146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078081.009154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078081.009159:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078081.009168:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3786 00000100:00080000:2.0:1769078081.009179:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078081.009186:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078081.009229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913579008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078081.009257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078081.009260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078081.009262:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078081.009267:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3786 00000100:00080000:2.0:1769078081.009271:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078081.009276:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078081.009289:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078081.009299:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913579008:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078081.009305:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913579136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078081.009314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078081.009316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078081.009318:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078081.009321:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3786 00000100:00080000:2.0:1769078081.009324:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078081.009327:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078081.009331:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078081.009339:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913579264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078081.009345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078081.009347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078081.009348:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078081.009350:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3786 00000100:00080000:2.0:1769078081.009353:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078081.009355:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078081.009358:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078081.009364:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913579392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078081.009370:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078081.009373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078081.009376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078081.009378:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078081.009382:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3786 00000100:00080000:2.0:1769078081.009386:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078081.009391:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078081.009396:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078081.009408:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913579520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078081.009416:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078081/real 1769078081] req@ffff935002a62a00 x1855008913579008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078081.009421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078081.009427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078081.009429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078081.009433:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913579008/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078081.009436:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3786 00000100:00080000:2.0:1769078081.009443:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078081.009447:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078081.009447:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769078081.009453:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769078081.009454:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078081.009459:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913579008:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078081.009466:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913579648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078081.009473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078081.009475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078081.009476:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078081.009479:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3786 00000100:00100000:0.0:1769078081.009481:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913579136:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769078081.009482:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078081.009489:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078081.009493:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078081.009503:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913579776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078081.009508:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078081.009523:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078081/real 1769078081] req@ffff935002a60000 x1855008913579136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078081.009538:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913579136/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078081.009547:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078081.009550:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078081.009555:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913579136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078081.009561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913579264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078081.009580:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078081.009588:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078081/real 1769078081] req@ffff935002a61500 x1855008913579264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078081.009595:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913579264/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078081.009603:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078081.009606:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078081.009609:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913579264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078081.009615:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913579392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078081.009633:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078081.009643:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078081/real 1769078081] req@ffff935002a63100 x1855008913579392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078081.009654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913579392/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078081.009659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078081.009662:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078081.009664:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913579392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078081.009667:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913579520:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078081.009679:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078081.009687:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078081/real 1769078081] req@ffff935002a61c00 x1855008913579520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078081.009695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913579520/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078081.009703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078081.009706:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078081.009709:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913579520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078081.009714:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913579648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078081.009732:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078081.009742:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078081/real 1769078081] req@ffff935002a61f80 x1855008913579648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078081.009752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913579648/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078081.009757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078081.009759:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078081.009761:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913579648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078081.009775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913579776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078081.009787:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078081.009793:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078081/real 1769078081] req@ffff935002a60a80 x1855008913579776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078081.009799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913579776/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078081.009804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078081.009806:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078081.009808:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913579776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078081.201160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078081.713190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078081.713191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078081.713195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078081.713232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078082.033215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078082.225144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078082.737148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078082.737164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078082.737172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078082.737225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078083.057172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078083.249135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078083.761138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078083.761138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078083.761159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078083.761221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078084.081163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078084.273142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078084.785150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078084.785150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078084.785150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078084.785254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078085.105116:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078085.297145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078085.809118:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078085.809118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078085.809142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078085.809207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078086.129179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078086.129188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078086.129192:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078086.129200:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3791 00000100:00100000:2.0:1769078086.129207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078086.129211:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078086.129223:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078086.129261:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913579904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078086.129281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078086.129284:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078086.129285:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078086.129288:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3791 00000100:00080000:1.0:1769078086.129290:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078086.129293:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078086.129300:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078086.129310:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913580032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078086.129316:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913579904:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078086.129318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078086.129324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078086.129326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078086.129329:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3791 00000100:00080000:1.0:1769078086.129331:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078086.129333:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078086.129336:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078086.129344:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913580160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078086.129354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078086.129357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078086.129359:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078086.129361:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3791 00000100:00080000:1.0:1769078086.129373:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078086.129375:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078086.129379:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078086.129386:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913580288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078086.129391:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078086.129397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078086.129400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078086.129401:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078086.129404:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3791 00000100:00080000:1.0:1769078086.129406:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078086.129409:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078086.129411:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078086.129418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913580416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078086.129423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078086.129426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078086.129428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078086.129431:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3791 00000100:00100000:0.0:1769078086.129432:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078086/real 1769078086] req@ffff9350100cb800 x1855008913579904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078086.129433:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078086.129438:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078086.129444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078086.129453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913580544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078086.129456:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913579904/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078086.129467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078086.129471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078086.129472:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078086.129473:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078086.129477:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078086.129481:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3791 00000100:00100000:0.0:1769078086.129482:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913579904:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078086.129484:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078086.129489:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078086.129492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078086.129499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913580672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078086.129506:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580032:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078086.129524:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078086.129533:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078086/real 1769078086] req@ffff9350100ca300 x1855008913580032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078086.129540:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913580032/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078086.129546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078086.129548:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078086.129550:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078086.129553:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078086.129564:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078086.129569:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078086/real 1769078086] req@ffff9350100c8e00 x1855008913580160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078086.129575:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913580160/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078086.129580:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078086.129582:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078086.129583:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078086.129587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078086.129595:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078086.129599:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078086/real 1769078086] req@ffff9350100ca680 x1855008913580288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078086.129610:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913580288/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078086.129615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078086.129616:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078086.129618:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078086.129620:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913580416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078086.129629:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078086.129634:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078086/real 1769078086] req@ffff9350100c8380 x1855008913580416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078086.129641:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913580416/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078086.129645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078086.129647:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078086.129648:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913580416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078086.129651:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913580544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078086.129659:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078086.129663:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078086/real 1769078086] req@ffff9350100caa00 x1855008913580544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078086.129669:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913580544/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078086.129678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078086.129680:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078086.129682:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913580544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078086.129691:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913580672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078086.129699:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078086.129703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078086/real 1769078086] req@ffff9350100c9500 x1855008913580672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078086.129710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913580672/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078086.129715:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078086.129716:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078086.129718:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913580672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078086.321138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078086.833206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078086.833206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078086.833207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078086.833245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078087.153225:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078087.345091:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078087.857132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078087.857133:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078087.857139:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078087.857245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078088.177160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078088.369137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078088.881137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078088.881137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078088.881137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078088.881164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078089.201165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078089.393166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078089.905116:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078089.905123:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078089.905139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078089.905163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078090.225160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078090.417173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078090.929144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078090.929144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078090.929160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078090.929178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078091.249213:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078091.249216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078091.249228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078091.249236:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078091.249248:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3796 00000100:00080000:3.0:1769078091.249262:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078091.249270:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078091.249334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913580800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078091.249367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078091.249371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078091.249373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078091.249378:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3796 00000100:00080000:3.0:1769078091.249382:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078091.249387:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078091.249399:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078091.249415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913580928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078091.249427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078091.249432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078091.249434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078091.249439:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3796 00000100:00080000:3.0:1769078091.249445:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078091.249451:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078091.249458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078091.249461:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913580800:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078091.249474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913581056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078091.249489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078091.249493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078091.249495:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078091.249501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3796 00000100:00080000:3.0:1769078091.249506:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078091.249512:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078091.249519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078091.249533:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913581184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078091.249546:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078091.249546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078091.249557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078091.249560:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078091.249567:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3796 00000100:00080000:3.0:1769078091.249581:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078091.249587:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078091.249594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078091.249603:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078091/real 1769078091] req@ffff935002a7fb80 x1855008913580800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078091.249614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913581312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078091.249628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078091.249629:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913580800/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078091.249637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078091.249643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078091.249655:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3796 00000100:00080000:0.0:1769078091.249656:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078091.249661:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078091.249666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078091.249672:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078091.249678:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913580800:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769078091.249683:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078091.249693:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078091.249701:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913581440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078091.249715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078091.249719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078091.249724:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769078091.249725:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078091.249738:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3796 00000100:00080000:3.0:1769078091.249746:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078091.249755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078091/real 1769078091] req@ffff935002a7df80 x1855008913580928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078091.249755:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078091.249778:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078091.249801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913581568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078091.249802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913580928/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078091.249853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078091.249858:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078091.249862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913580928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078091.249894:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078091.249917:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078091.249928:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078091/real 1769078091] req@ffff935002a7f800 x1855008913581056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078091.249948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913581056/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078091.249957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078091.249961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078091.249965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078091.249971:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078091.249988:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078091.249998:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078091/real 1769078091] req@ffff935002a7c000 x1855008913581184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078091.250010:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913581184/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078091.250018:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078091.250022:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078091.250025:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078091.250031:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913581312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078091.250092:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078091.250105:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078091/real 1769078091] req@ffff935002a7c700 x1855008913581312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078091.250115:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913581312/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078091.250124:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078091.250127:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078091.250129:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913581312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078091.250133:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913581440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078091.250147:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078091.250153:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078091/real 1769078091] req@ffff935002a7ca80 x1855008913581440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078091.250160:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913581440/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078091.250165:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078091.250168:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078091.250169:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913581440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078091.250173:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913581568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078091.250181:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078091.250186:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078091/real 1769078091] req@ffff935002a7f100 x1855008913581568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078091.250192:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913581568/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078091.250197:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078091.250198:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078091.250201:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913581568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078091.441219:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078091.953197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078091.953197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078091.953232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078091.953255:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078092.273185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078092.465203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078092.977187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078092.977188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078092.977198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078092.977218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078093.297168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078093.489153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078094.001200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078094.001220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078094.001227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078094.001341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078094.321194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078094.513146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078095.025188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078095.025188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078095.025193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078095.025208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078095.345144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078095.537212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078096.049199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078096.049199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078096.049207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078096.049238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:0.0:1769078096.369182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078096.369192:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078096.369197:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078096.369206:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3802 00000100:00100000:1.0:1769078096.369209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078096.369216:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078096.369224:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078096.369258:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913581696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078096.369283:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078096.369286:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078096.369287:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078096.369290:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3802 00000100:00080000:0.0:1769078096.369293:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078096.369295:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078096.369303:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078096.369313:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913581824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078096.369321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078096.369325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078096.369327:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078096.369328:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913581696:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078096.369331:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3802 00000100:00080000:0.0:1769078096.369335:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078096.369339:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078096.369343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078096.369353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913581952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078096.369359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078096.369361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078096.369362:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078096.369365:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3802 00000100:00080000:0.0:1769078096.369367:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078096.369369:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078096.369373:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078096.369380:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913582080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078096.369386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078096.369388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078096.369391:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078096.369393:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3802 00000100:00080000:0.0:1769078096.369396:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078096.369398:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078096.369402:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078096.369407:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078096.369412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913582208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078096.369419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078096.369422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078096.369424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078096.369427:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3802 00000100:00080000:0.0:1769078096.369429:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078096.369431:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078096.369433:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078096.369442:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913582336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078096.369447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078096.369449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078096.369450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078096.369452:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3802 00000100:00080000:0.0:1769078096.369454:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078096.369457:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078096.369461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078096.369466:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078096/real 1769078096] req@ffff9350103faa00 x1855008913581696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078096.369470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913582464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078096.369519:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913581696/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078096.369532:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078096.369536:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078096.369540:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913581696:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078096.369563:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078096.369586:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078096.369598:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078096/real 1769078096] req@ffff9350103f8700 x1855008913581824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078096.369611:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913581824/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078096.369621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078096.369625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078096.369629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078096.369635:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078096.369653:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078096.369662:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078096/real 1769078096] req@ffff9350103fb800 x1855008913581952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078096.369675:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913581952/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078096.369701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078096.369705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078096.369708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913581952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078096.369714:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078096.369733:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078096.369743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078096/real 1769078096] req@ffff9350103fb480 x1855008913582080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078096.369755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913582080/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078096.369763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078096.369767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078096.369770:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078096.369775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913582208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078096.369790:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078096.369798:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078096/real 1769078096] req@ffff9350103fad80 x1855008913582208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078096.369810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913582208/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078096.369818:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078096.369822:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078096.369825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913582208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078096.369830:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913582336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078096.369844:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078096.369852:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078096/real 1769078096] req@ffff9350103fb100 x1855008913582336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078096.369863:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913582336/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078096.369871:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078096.369874:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078096.369877:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913582336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078096.369883:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913582464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078096.369896:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078096.369904:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078096/real 1769078096] req@ffff9350103f8380 x1855008913582464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078096.369930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913582464/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078096.369939:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078096.369942:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078096.369945:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913582464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078096.561175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078097.073165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078097.073182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078097.073188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078097.073199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078097.393167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078097.585192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078098.097196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078098.097196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078098.097200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078098.097251:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078098.417192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078098.609174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078099.121126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078099.121163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078099.121164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078099.121164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078099.441165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078099.633162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078100.145161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078100.145161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078100.145168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078100.145198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078100.465173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078100.657146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078101.169193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078101.169194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078101.169206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078101.169269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078101.489207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078101.489211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078101.489224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078101.489229:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078101.489243:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3807 00000100:00080000:1.0:1769078101.489255:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078101.489264:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078101.489310:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913582592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078101.489342:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078101.489348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078101.489350:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078101.489356:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3807 00000100:00080000:1.0:1769078101.489360:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078101.489364:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078101.489376:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078101.489388:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913582592:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078101.489394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913582720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078101.489406:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078101.489408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078101.489410:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078101.489416:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3807 00000100:00080000:1.0:1769078101.489419:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078101.489425:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078101.489431:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078101.489447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913582848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078101.489460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078101.489465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078101.489467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078101.489474:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078101.489479:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3807 00000100:00080000:1.0:1769078101.489485:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078101.489490:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078101.489497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078101.489510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913582976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078101.489519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078101.489525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1769078101.489527:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078101/real 1769078101] req@ffff9350100c9880 x1855008913582592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078101.489534:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078101.489552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3807 00000100:00080000:1.0:1769078101.489557:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078101.489562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913582592/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078101.489573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078101.489574:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078101.489585:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:1.0:1769078101.489588:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078101.489608:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913582592:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078101.489628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913583104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078101.489637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078101.489640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078101.489643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078101.489647:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582720:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078101.489648:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3807 00000100:00080000:1.0:1769078101.489657:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078101.489664:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078101.489668:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078101.489678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913583232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078101.489683:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078101.489697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078101.489701:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078101/real 1769078101] req@ffff9350100cad80 x1855008913582720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078101.489702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078101.489706:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078101.489720:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3807 00000100:00080000:1.0:1769078101.489728:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078101.489735:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078101.489741:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913582720/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769078101.489742:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078101.489761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769078101.489762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913583360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078101.489778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078101.489792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078101.489803:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078101.489835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078101.489847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078101/real 1769078101] req@ffff9350100c9c00 x1855008913582848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078101.489860:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913582848/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078101.489870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078101.489874:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078101.489876:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078101.489882:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078101.489898:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078101.489905:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078101/real 1769078101] req@ffff9350100cb100 x1855008913582976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078101.489918:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913582976/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078101.489924:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078101.489926:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078101.489928:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913582976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078101.489932:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913583104:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078101.489943:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078101.489948:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078101/real 1769078101] req@ffff9350100c9180 x1855008913583104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078101.489956:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913583104/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078101.489961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078101.489963:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078101.489965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913583104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078101.489983:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913583232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078101.490003:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078101.490014:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078101/real 1769078101] req@ffff9350100c8a80 x1855008913583232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078101.490026:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913583232/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078101.490039:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078101.490042:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078101.490045:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913583232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078101.490077:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913583360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078101.490098:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078101.490109:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078101/real 1769078101] req@ffff9350100c8000 x1855008913583360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078101.490123:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913583360/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078101.490132:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078101.490135:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078101.490137:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913583360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078101.681156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078102.193160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078102.193199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078102.193199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078102.193221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078102.513223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078102.705212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078103.217171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078103.217171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078103.217241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078103.217285:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078103.537214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078103.729145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078104.241171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078104.241171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078104.241173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078104.241218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078104.561150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078104.753201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078105.265177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078105.265177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078105.265199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078105.265248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078105.585182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078105.777157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078106.289178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078106.289202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078106.289202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078106.289252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078106.609140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078106.609140:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078106.609150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078106.609154:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078106.609163:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3812 00000100:00080000:3.0:1769078106.609174:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078106.609180:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078106.609223:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913583488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078106.609244:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078106.609247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078106.609248:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078106.609252:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3812 00000100:00080000:3.0:1769078106.609254:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078106.609257:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078106.609264:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078106.609272:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913583616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078106.609280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078106.609282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078106.609284:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078106.609287:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3812 00000100:00080000:3.0:1769078106.609290:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078106.609294:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078106.609298:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078106.609304:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913583744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078106.609311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078106.609314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078106.609316:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078106.609319:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3812 00000100:00080000:3.0:1769078106.609321:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078106.609323:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078106.609326:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078106.609337:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913583488:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078106.609338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913583872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078106.609360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078106.609362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078106.609363:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078106.609366:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3812 00000100:00080000:3.0:1769078106.609368:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078106.609373:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078106.609379:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078106.609394:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913584000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078106.609405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078106.609410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078106.609413:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078106.609417:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3812 00000100:00080000:3.0:1769078106.609433:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078106.609438:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078106.609442:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769078106.609452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078106.609468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913584128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078106.609476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078106.609479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078106.609480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078106.609484:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3812 00000100:00080000:3.0:1769078106.609487:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078106.609490:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078106.609495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078106.609503:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078106/real 1769078106] req@ffff935002a7dc00 x1855008913583488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078106.609517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913584256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078106.609565:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913583488/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078106.609575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078106.609581:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078106.609584:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913583488:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078106.609592:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913583616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078106.609614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078106.609623:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078106/real 1769078106] req@ffff935002a7d180 x1855008913583616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078106.609633:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913583616/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078106.609642:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078106.609646:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078106.609650:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913583616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078106.609657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913583744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078106.609679:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078106.609691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078106/real 1769078106] req@ffff935002a7e300 x1855008913583744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078106.609704:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913583744/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078106.609711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078106.609714:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078106.609718:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913583744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078106.609751:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913583872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078106.609774:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078106.609787:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078106/real 1769078106] req@ffff935002a7c380 x1855008913583872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078106.609800:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913583872/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078106.609809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078106.609812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078106.609816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913583872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078106.609821:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584000:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078106.609837:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078106.609844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078106/real 1769078106] req@ffff935002a7f100 x1855008913584000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078106.609852:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913584000/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078106.609858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078106.609860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078106.609862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078106.609866:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078106.609876:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078106.609881:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078106/real 1769078106] req@ffff935002a7ca80 x1855008913584128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078106.609894:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913584128/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078106.609899:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078106.609901:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078106.609903:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078106.609906:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078106.609915:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078106.609921:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078106/real 1769078106] req@ffff935002a7c700 x1855008913584256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078106.609928:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913584256/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078106.609934:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078106.609935:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078106.609937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078106.801174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078107.313134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078107.313134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078107.313154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078107.313194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078107.633260:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078107.825213:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078108.337185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078108.337191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078108.337209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078108.337246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078108.657176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078108.849272:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078109.361131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078109.361131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078109.361146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078109.361207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078109.681150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078109.873218:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078110.385145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078110.385145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078110.385156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078110.385237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078110.705213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078110.897228:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078111.409124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078111.409138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078111.409140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078111.409281:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078111.729166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078111.729287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078111.729292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078111.729297:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078111.729306:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3817 00000100:00080000:3.0:1769078111.729318:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078111.729326:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078111.729386:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913584384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078111.729412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078111.729415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078111.729416:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078111.729430:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3817 00000100:00080000:3.0:1769078111.729433:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078111.729437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078111.729447:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078111.729462:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913584512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078111.729472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078111.729476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078111.729478:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078111.729482:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3817 00000100:00080000:3.0:1769078111.729486:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078111.729491:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078111.729498:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078111.729511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913584640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078111.729524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078111.729527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078111.729528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078111.729533:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3817 00000100:00080000:3.0:1769078111.729537:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078111.729541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078111.729546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078111.729560:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913584384:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078111.729560:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913584768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078111.729574:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078111.729582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078111.729585:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078111.729593:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3817 00000100:00080000:3.0:1769078111.729596:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078111.729599:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078111.729604:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078111.729615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913584896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078111.729622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078111.729624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078111.729625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078111.729628:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3817 00000100:00080000:3.0:1769078111.729631:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078111.729633:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078111.729636:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078111.729642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913585024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078111.729650:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078111.729652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078111.729653:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078111.729655:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3817 00000100:00080000:3.0:1769078111.729657:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078111.729659:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078111.729662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078111.729667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913585152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078111.729776:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078111.729861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078111/real 1769078111] req@ffff935002a7c000 x1855008913584384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078111.729893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913584384/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078111.729906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078111.729922:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078111.729925:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913584384:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078111.729937:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913584512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078111.729971:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078111.730004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078111/real 1769078111] req@ffff935002a7f800 x1855008913584512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078111.730031:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913584512/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078111.730042:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078111.730046:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078111.730159:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913584512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078111.730214:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913584640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078111.730263:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078111.730275:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078111/real 1769078111] req@ffff935002a7df80 x1855008913584640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078111.730284:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913584640/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078111.730290:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078111.730292:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078111.730295:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913584640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078111.730300:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913584768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078111.730312:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078111.730319:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078111/real 1769078111] req@ffff935002a7fb80 x1855008913584768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078111.730350:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913584768/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078111.730360:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078111.730364:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078111.730368:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913584768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078111.730392:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584896:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078111.730444:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078111.730470:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078111/real 1769078111] req@ffff935002a7c700 x1855008913584896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078111.730479:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913584896/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078111.730485:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078111.730488:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078111.730490:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913584896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078111.730508:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078111.730524:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078111.730535:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078111/real 1769078111] req@ffff935002a7ca80 x1855008913585024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078111.730547:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913585024/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078111.730556:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078111.730560:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078111.730563:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078111.730571:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078111.730613:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078111.730644:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078111/real 1769078111] req@ffff935002a7f100 x1855008913585152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078111.730654:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913585152/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078111.730659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078111.730667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078111.730677:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078111.921212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078112.433192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078112.433199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078112.433227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078112.433254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078112.753188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078112.945216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078113.457208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078113.457208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078113.457209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078113.457247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078113.777212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078113.969158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078114.481187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078114.481187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078114.481214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078114.481248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078114.801212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078114.993212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078115.505213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078115.505213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078115.505215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078115.505268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078115.825237:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078116.017165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078116.529132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078116.529132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078116.529163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078116.529187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078116.849175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078116.849196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078116.849207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078116.849213:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.849223:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3822 00000100:00080000:0.0:1769078116.849234:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.849240:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078116.849282:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913585280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078116.849311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078116.849314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078116.849316:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.849320:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3822 00000100:00080000:0.0:1769078116.849324:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.849328:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078116.849367:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078116.849377:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913585280:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078116.849384:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913585408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078116.849394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078116.849397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078116.849399:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.849403:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3822 00000100:00080000:0.0:1769078116.849408:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.849413:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078116.849420:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078116.849434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913585536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078116.849445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078116.849449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078116.849451:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.849455:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3822 00000100:00080000:0.0:1769078116.849459:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.849462:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078116.849467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078116.849479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913585664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078116.849486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078116.849489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078116.849490:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.849493:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3822 00000100:00080000:0.0:1769078116.849496:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078116.849503:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913585408:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078116.849505:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078116.849517:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078116.849533:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913585792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078116.849541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078116.849544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078116.849545:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.849549:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3822 00000100:00100000:2.0:1769078116.849554:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913585536:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769078116.849556:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.849567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078116.849573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078116.849583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913585664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078116.849594:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913585920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078116.849609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078116.849610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585792:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078116.849623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078116.849626:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.849630:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3822 00000100:00080000:0.0:1769078116.849637:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.849642:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078116.849648:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078116.849648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078116.849675:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913586048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078116.849708:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078117.041161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078117.553179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078117.553218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078117.553319:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078117.553359:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078117.873284:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078118.065133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078118.577183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078118.577184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078118.577194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078118.577260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078118.897156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078119.089170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078119.601145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078119.601149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078119.601183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078119.601207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078119.921131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769078119.921279:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769078119.921331:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078119.921345:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078119.921374:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078119.921383:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078119.921393:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078119.921404:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078119.921414:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078119.921423:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078116/real 1769078119] req@ffff9350103f9180 x1855008913585792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769078119.921425:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078119.921451:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913585792/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078119.921462:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078119.921467:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078119.921472:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078119.921480:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078116/real 1769078119] req@ffff9350103f9c00 x1855008913585408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078119.921490:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913585408/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078119.921498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078119.921501:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078119.921503:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913585408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078119.921510:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078116/real 1769078119] req@ffff9350103fbb80 x1855008913585536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078119.921521:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913585536/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078119.921527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078119.921531:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078119.921533:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913585536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078119.921540:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078116/real 1769078119] req@ffff9350103f9500 x1855008913585664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078119.921550:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913585664/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078119.921558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078119.921561:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078119.921564:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913585664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078119.921569:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078116/real 1769078119] req@ffff9350103f9f80 x1855008913585280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078119.921579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913585280/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078119.921585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078119.921588:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078119.921591:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913585280:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078119.921622:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078116/real 1769078119] req@ffff9350103fb100 x1855008913586048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078119.921633:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913586048/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078119.921640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078119.921643:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078119.921646:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078119.921652:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078116/real 1769078119] req@ffff9350103f8380 x1855008913585920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078119.921661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913585920/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078119.921668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078119.921671:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078119.921674:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913585920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078120.113161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078120.625154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078120.625154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078120.625154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078120.625201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078120.945180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078121.137141:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078121.649136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078121.649168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078121.649168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078121.649216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078121.969152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078121.969158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078121.969163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078121.969170:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.969180:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3827 00000100:00080000:0.0:1769078121.969191:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.969199:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078121.969236:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913586176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.969271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.969274:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.969275:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.969278:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3827 00000100:00080000:0.0:1769078121.969282:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.969285:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078121.969296:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078121.969309:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913586304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.969318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078121.969320:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913586176:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078121.969322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.969325:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.969334:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3827 00000100:00080000:0.0:1769078121.969336:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.969338:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078121.969342:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078121.969351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913586432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.969361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.969364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078121.969366:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.969369:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3827 00000100:00080000:0.0:1769078121.969371:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.969374:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078121.969377:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078121.969384:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913586560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.969395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.969397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.969399:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.969402:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3827 00000800:00000100:2.0:1769078121.969408:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078121.969409:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.969417:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078121.969422:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078121.969434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913586688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.969444:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.969448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.969450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.969454:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3827 00000100:00080000:0.0:1769078121.969459:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.969464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078121.969470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078121.969478:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9350103fad80 x1855008913586176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078121.969485:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913586816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078121.969510:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913586176/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078121.969514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.969523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078121.969528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.969535:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3827 00000100:00080000:2.0:1769078121.969537:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078121.969540:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078121.969547:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769078121.969548:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078121.969558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078121.969560:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913586176:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078121.969574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913586944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078121.969595:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913586304:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078121.969628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078121.969645:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9350103fb480 x1855008913586304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.969658:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913586304/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.969667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.969670:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078121.969674:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913586304:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078121.969680:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913586432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078121.969702:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078121.969714:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9350103fb800 x1855008913586432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.969727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913586432/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.969737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.969741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078121.969745:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913586432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078121.969751:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913586560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078121.969771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078121.969782:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9350103f8700 x1855008913586560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.969807:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913586560/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.969816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.969819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078121.969823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913586560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078121.969830:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586688:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078121.969851:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078121.969861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9350103faa00 x1855008913586688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.969873:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913586688/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.969881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.969884:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078121.969888:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586688:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078121.969894:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078121.969912:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078121.969923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9350103f8380 x1855008913586816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.969936:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913586816/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.969945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.969948:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078121.969952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078121.969956:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078121.969969:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078121.969979:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9350103fb100 x1855008913586944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.969991:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913586944/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.969999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.970002:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078121.970012:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913586944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078122.161180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078122.673174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078122.673194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078122.673210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078122.673275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078122.993173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078123.185166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078123.697170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078123.697170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078123.697171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078123.697235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078124.017142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078124.209173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078124.721149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078124.721150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078124.721150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078124.721269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078125.041156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078125.233140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078125.745163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078125.745163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078125.745163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078125.745243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078126.065207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078126.257233:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078126.769141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078126.769142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078126.769148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078126.769229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078127.089219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078127.089229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078127.089234:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078127.089243:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3832 00000100:00100000:1.0:1769078127.089251:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078127.089254:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078127.089266:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078127.089301:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913587072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078127.089327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078127.089331:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078127.089332:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078127.089335:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3832 00000100:00080000:0.0:1769078127.089338:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078127.089340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078127.089348:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078127.089363:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913587200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078127.089377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078127.089385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769078127.089385:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913587072:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078127.089394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078127.089401:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3832 00000100:00080000:0.0:1769078127.089406:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078127.089413:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078127.089419:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078127.089433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913587328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078127.089441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078127.089444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078127.089445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078127.089448:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3832 00000100:00080000:0.0:1769078127.089451:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078127.089454:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078127.089458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078127.089466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913587456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078127.089474:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078127.089485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078127.089489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078127.089491:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078127.089497:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3832 00000100:00080000:0.0:1769078127.089501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078127.089504:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078127.089509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078127.089517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913587584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078127.089526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078127.089530:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078127/real 1769078127] req@ffff9350103f9f80 x1855008913587072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078127.089531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078127.089536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078127.089549:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3832 00000100:00080000:0.0:1769078127.089557:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078127.089567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078127.089572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078127.089578:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913587072/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078127.089589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913587712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078127.089597:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078127.089601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078127.089607:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769078127.089608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078127.089614:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078127.089619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913587072:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078127.089626:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3832 00000100:00080000:0.0:1769078127.089630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078127.089634:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078127.089639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078127.089649:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913587840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078127.089655:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913587200:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078127.089694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078127.089709:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078127/real 1769078127] req@ffff9350103f9500 x1855008913587200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078127.089726:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913587200/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078127.089733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078127.089737:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078127.089741:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913587200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078127.089747:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913587328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078127.089760:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078127.089768:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078127/real 1769078127] req@ffff9350103fbb80 x1855008913587328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078127.089776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913587328/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078127.089783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078127.089786:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078127.089788:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913587328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078127.089793:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913587456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078127.089802:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078127.089808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078127/real 1769078127] req@ffff9350103f9c00 x1855008913587456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078127.089815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913587456/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078127.089820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078127.089822:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078127.089824:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913587456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078127.089827:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913587584:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078127.089837:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078127.089842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078127/real 1769078127] req@ffff9350103f9180 x1855008913587584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078127.089849:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913587584/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078127.089854:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078127.089856:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078127.089858:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913587584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078127.089861:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913587712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078127.089870:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078127.089875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078127/real 1769078127] req@ffff9350103fb100 x1855008913587712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078127.089883:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913587712/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078127.089888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078127.089890:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078127.089892:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913587712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078127.089905:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913587840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078127.089914:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078127.089920:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078127/real 1769078127] req@ffff9350103f8380 x1855008913587840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078127.089927:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913587840/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078127.089932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078127.089934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078127.089936:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913587840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078127.281150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078127.793191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078127.793191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078127.793196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078127.793231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078128.113243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078128.305208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078128.817154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078128.817155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078128.817216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078128.817294:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078129.137219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078129.329202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078129.841158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078129.841162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078129.841162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078129.841215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078130.161259:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078130.353146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078130.865187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078130.865187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078130.865187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078130.865222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078131.185177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078131.377244:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078131.889161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078131.889178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078131.889180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078131.889219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078132.209223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078132.209226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078132.209236:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078132.209241:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078132.209251:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3837 00000100:00080000:1.0:1769078132.209262:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078132.209269:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078132.209311:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913587968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078132.209334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078132.209337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078132.209338:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078132.209342:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3837 00000100:00080000:1.0:1769078132.209344:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078132.209346:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078132.209356:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078132.209368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913588096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078132.209375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078132.209378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078132.209380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078132.209383:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3837 00000100:00080000:1.0:1769078132.209385:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078132.209388:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078132.209393:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078132.209399:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913587968:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078132.209410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913588224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078132.209418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078132.209421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078132.209422:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078132.209425:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3837 00000100:00080000:1.0:1769078132.209428:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078132.209430:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078132.209433:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078132.209441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913588352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078132.209447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078132.209450:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078132.209451:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078132.209453:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3837 00000100:00080000:1.0:1769078132.209456:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078132.209458:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078132.209461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078132.209467:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913588480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078132.209475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078132.209478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078132.209481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078132.209487:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078132.209499:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3837 00000100:00080000:1.0:1769078132.209505:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078132.209509:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078132.209515:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078132.209534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913588608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078132.209543:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078132/real 1769078132] req@ffff9350100c9f80 x1855008913587968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078132.209546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078132.209557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078132.209561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078132.209571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3837 00000100:00080000:1.0:1769078132.209577:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078132.209583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078132.209588:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078132.209591:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913587968/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078132.209603:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913588736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078132.209607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078132.209618:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078132.209624:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913587968:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078132.209634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588096:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078132.209658:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078132.209684:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078132/real 1769078132] req@ffff9350100c8000 x1855008913588096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078132.209699:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913588096/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078132.209708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078132.209711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078132.209715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078132.209744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078132.209766:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078132.209778:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078132/real 1769078132] req@ffff9350100c8a80 x1855008913588224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078132.209790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913588224/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078132.209805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078132.209809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078132.209812:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078132.209818:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078132.209838:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078132.209848:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078132/real 1769078132] req@ffff9350100c9180 x1855008913588352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078132.209859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913588352/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078132.209868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078132.209871:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078132.209875:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078132.209881:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913588480:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078132.209900:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078132.209923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078132/real 1769078132] req@ffff9350100cb100 x1855008913588480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078132.209935:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913588480/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078132.209943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078132.209946:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078132.209950:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913588480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078132.209957:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913588608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078132.209976:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078132.209987:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078132/real 1769078132] req@ffff9350100c9c00 x1855008913588608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078132.209999:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913588608/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078132.210007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078132.210010:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078132.210014:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913588608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078132.210020:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913588736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078132.210040:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078132.210094:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078132/real 1769078132] req@ffff9350100cad80 x1855008913588736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078132.210108:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913588736/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078132.210116:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078132.210120:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078132.210123:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913588736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078132.401193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078132.913197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078132.913237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078132.913253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078132.913342:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078133.233146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078133.425185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078133.937136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078133.937136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078133.937173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078133.937206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078134.257232:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078134.449193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078134.961233:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078134.961244:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078134.961244:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078134.961276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078135.281185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078135.473143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078135.985129:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078135.985129:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078135.985131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078135.985166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078136.305212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078136.497196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078137.009192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078137.009193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078137.009205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078137.009230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078137.329154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078137.329167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078137.329172:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078137.329183:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3843 00000100:00100000:0.0:1769078137.329185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078137.329194:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078137.329202:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078137.329236:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913588864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078137.329262:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078137.329265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078137.329267:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078137.329270:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3843 00000100:00080000:1.0:1769078137.329272:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078137.329275:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078137.329282:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078137.329291:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913588992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078137.329298:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078137.329301:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078137.329302:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078137.329305:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3843 00000100:00080000:1.0:1769078137.329307:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078137.329309:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078137.329312:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078137.329322:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913589120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078137.329322:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913588864:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078137.329337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078137.329339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078137.329340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078137.329343:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3843 00000100:00080000:1.0:1769078137.329345:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078137.329348:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078137.329351:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078137.329361:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913589248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078137.329368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078137.329371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078137.329372:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078137.329374:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3843 00000100:00080000:1.0:1769078137.329377:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078137.329380:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078137.329382:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078137.329389:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769078137.329392:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913589376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078137.329400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078137.329403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078137.329404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078137.329407:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3843 00000100:00080000:1.0:1769078137.329411:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078137.329413:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078137.329417:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078137.329429:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078137/real 1769078137] req@ffff9350100c9880 x1855008913588864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078137.329430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913589504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078137.329465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078137.329469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078137.329474:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078137.329474:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913588864/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078137.329487:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078137.329493:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078137.329494:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3843 00000100:00100000:2.0:1769078137.329498:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913588864:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078137.329505:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588992:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078137.329506:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078137.329515:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078137.329522:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078137.329526:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078137.329543:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078137/real 1769078137] req@ffff9350100c9500 x1855008913588992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078137.329544:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913589632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078137.329573:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913588992/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078137.329585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078137.329589:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078137.329594:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913588992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078137.329621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913589120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078137.329643:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078137.329653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078137/real 1769078137] req@ffff9350100caa00 x1855008913589120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078137.329661:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913589120/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078137.329676:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078137.329679:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078137.329681:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913589120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078137.329685:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913589248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078137.329698:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078137.329705:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078137/real 1769078137] req@ffff9350100c8380 x1855008913589248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078137.329712:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913589248/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078137.329718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078137.329720:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078137.329722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913589248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078137.329725:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913589376:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078137.329735:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078137.329740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078137/real 1769078137] req@ffff9350100ca680 x1855008913589376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078137.329750:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913589376/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078137.329759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078137.329762:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078137.329765:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913589376:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078137.329771:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913589504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078137.329791:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078137.329803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078137/real 1769078137] req@ffff9350100c8e00 x1855008913589504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078137.329815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913589504/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078137.329820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078137.329822:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078137.329825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913589504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078137.329829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913589632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078137.329842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078137.329848:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078137/real 1769078137] req@ffff9350100ca300 x1855008913589632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078137.329855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913589632/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078137.329865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078137.329867:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078137.329869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913589632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078137.521140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078138.033154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078138.033154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078138.033216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078138.033257:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078138.353160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078138.545209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078139.057174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078139.057174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078139.057212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078139.057222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078139.377167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078139.569119:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078140.081186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078140.081203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078140.081211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078140.081265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078140.401133:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078140.593150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078141.105186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078141.105202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078141.105252:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078141.105284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078141.425186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078141.617198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078142.129135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078142.129137:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078142.129190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078142.129252:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078142.449183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078142.449210:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078142.449222:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078142.449228:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078142.449239:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3848 00000100:00080000:2.0:1769078142.449250:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078142.449257:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078142.449295:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913589760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078142.449330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078142.449334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078142.449337:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078142.449343:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3848 00000100:00080000:2.0:1769078142.449350:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078142.449354:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078142.449363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078142.449370:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913589760:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078142.449379:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913589888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078142.449393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078142.449396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078142.449397:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078142.449402:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3848 00000100:00080000:2.0:1769078142.449405:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078142.449407:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078142.449412:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078142.449421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913590016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078142.449428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078142.449430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078142.449431:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078142.449435:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3848 00000100:00080000:2.0:1769078142.449437:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078142.449440:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078142.449442:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078142.449450:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913590144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078142.449456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078142.449459:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078142.449460:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078142.449471:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3848 00000800:00000100:0.0:1769078142.449472:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078142.449479:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078142.449483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078142.449490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078142.449503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913590272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078142.449510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078142.449513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078142.449514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078142.449519:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3848 00000100:00080000:2.0:1769078142.449525:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078142.449530:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078142.449534:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078142/real 1769078142] req@ffff935002a62300 x1855008913589760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769078142.449536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078142.449552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913590400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078142.449560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078142.449562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913589760/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078142.449564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078142.449568:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078142.449579:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078142.449583:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3848 00000100:00080000:0.0:1769078142.449590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769078142.449593:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078142.449595:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913589760:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078142.449605:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078142.449613:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078142.449626:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913589888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078142.449633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913590528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078142.449651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078142.449667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078142/real 1769078142] req@ffff935002a61880 x1855008913589888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078142.449676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913589888/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078142.449685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078142.449691:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078142.449693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913589888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078142.449698:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078142.449710:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078142.449716:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078142/real 1769078142] req@ffff935002a61180 x1855008913590016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078142.449728:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913590016/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078142.449733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078142.449734:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078142.449736:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078142.449740:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078142.449750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078142.449755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078142/real 1769078142] req@ffff935002a62680 x1855008913590144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078142.449762:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913590144/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078142.449768:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078142.449770:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078142.449772:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078142.449775:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913590272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078142.449785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078142.449791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078142/real 1769078142] req@ffff935002a60a80 x1855008913590272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078142.449798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913590272/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078142.449807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078142.449809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078142.449811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913590272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078142.449815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913590400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078142.449824:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078142.449830:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078142/real 1769078142] req@ffff935002a61f80 x1855008913590400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078142.449836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913590400/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078142.449841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078142.449843:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078142.449845:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913590400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078142.449859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913590528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078142.449868:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078142.449874:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078142/real 1769078142] req@ffff935002a61c00 x1855008913590528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078142.449900:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913590528/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078142.449922:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078142.449926:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078142.449930:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913590528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078142.641188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078143.153203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078143.153219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078143.153240:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078143.153263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078143.473167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078143.665197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078144.177150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078144.177163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078144.177273:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078144.177311:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078144.497243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078144.689124:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078145.201154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078145.201170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078145.201187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078145.201208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078145.521113:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078145.713194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078146.225157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078146.225157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078146.225163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078146.225219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078146.545188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078146.737158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078147.249175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078147.249175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078147.249177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078147.249271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078147.569233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078147.569234:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078147.569247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078147.569253:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078147.569265:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3853 00000100:00080000:3.0:1769078147.569281:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078147.569290:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078147.569348:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913590656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078147.569376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078147.569381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078147.569382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078147.569387:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3853 00000100:00080000:3.0:1769078147.569392:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078147.569397:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078147.569410:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078147.569422:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913590656:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078147.569426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913590784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078147.569447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078147.569451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078147.569452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078147.569457:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3853 00000100:00080000:3.0:1769078147.569461:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078147.569465:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078147.569471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078147.569481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913590912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078147.569504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078147.569508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078147.569510:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078147.569514:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3853 00000100:00080000:3.0:1769078147.569518:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078147.569522:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078147.569526:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078147.569533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078147.569544:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913591040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078147.569553:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078147.569557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078147.569558:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078147.569562:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3853 00000100:00080000:3.0:1769078147.569566:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078147.569571:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078147.569584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078147.569585:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078147/real 1769078147] req@ffff935002a7c380 x1855008913590656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078147.569601:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913591168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078147.569611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078147.569614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078147.569617:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078147.569622:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3853 00000100:00100000:0.0:1769078147.569623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913590656/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078147.569627:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078147.569635:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078147.569643:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769078147.569651:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078147.569656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913591296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078147.569661:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078147.569671:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913590656:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078147.569672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078147.569681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078147.569683:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078147.569688:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3853 00000100:00080000:3.0:1769078147.569691:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078147.569695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078147.569699:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078147.569709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913591424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078147.569710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590784:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078147.569746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078147.569763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078147/real 1769078147] req@ffff935002a7e300 x1855008913590784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078147.569784:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913590784/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078147.569794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078147.569797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078147.569801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078147.569807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078147.569868:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078147.569881:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078147/real 1769078147] req@ffff935002a7d180 x1855008913590912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078147.569895:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913590912/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078147.569904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078147.569907:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078147.569911:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913590912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078147.569917:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078147.569936:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078147.569947:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078147/real 1769078147] req@ffff935002a7dc00 x1855008913591040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078147.569960:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913591040/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078147.569975:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078147.569978:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078147.569982:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078147.569988:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913591168:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078147.570006:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078147.570018:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078147/real 1769078147] req@ffff935002a7f100 x1855008913591168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078147.570030:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913591168/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078147.570038:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078147.570042:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078147.570045:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913591168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078147.570073:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913591296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078147.570096:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078147.570109:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078147/real 1769078147] req@ffff935002a7ca80 x1855008913591296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078147.570122:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913591296/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078147.570130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078147.570134:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078147.570138:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913591296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078147.570163:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913591424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078147.570183:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078147.570195:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078147/real 1769078147] req@ffff935002a7c700 x1855008913591424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078147.570207:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913591424/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078147.570214:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078147.570218:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078147.570221:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913591424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078147.761188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078148.273182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078148.273182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078148.273182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078148.273222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078148.593193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078148.785193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078149.297178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078149.297178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078149.297178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078149.297243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078149.617247:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078149.809191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078150.321191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078150.321191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078150.321192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078150.321229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078150.641139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078150.833187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078151.345142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078151.345143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078151.345298:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078151.345301:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078151.665201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078151.857177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078152.369138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078152.369142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078152.369178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078152.369189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078152.689170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078152.689180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078152.689184:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078152.689196:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3858 00000100:00080000:0.0:1769078152.689208:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078152.689215:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078152.689257:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913591552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078152.689283:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078152.689286:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078152.689287:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078152.689290:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3858 00000100:00080000:0.0:1769078152.689292:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078152.689295:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078152.689304:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078152.689313:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913591680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078152.689319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078152.689321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078152.689322:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078152.689325:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3858 00000100:00080000:0.0:1769078152.689327:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078152.689329:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078152.689332:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078152.689338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913591808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078152.689344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078152.689347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078152.689349:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078152.689351:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3858 00000100:00080000:0.0:1769078152.689354:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078152.689356:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078152.689359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078152.689365:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913591936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078152.689371:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913591552:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078152.689374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078152.689380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078152.689381:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078152.689385:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3858 00000100:00080000:0.0:1769078152.689387:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078152.689389:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078152.689392:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078152.689400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913592064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078152.689406:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078152.689408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078152.689409:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078152.689412:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3858 00000100:00080000:0.0:1769078152.689414:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078152.689416:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078152.689420:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078152.689428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913592192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078152.689433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078152.689436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078152.689437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078152.689440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3858 00000100:00080000:0.0:1769078152.689442:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078152.689444:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078152.689447:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078152.689451:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078152.689457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913592320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078152.689494:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769078152.689500:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078152/real 1769078152] req@ffff9350103faa00 x1855008913591552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078152.689517:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913591552/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078152.689526:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078152.689540:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078152.689544:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913591552:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078152.689549:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591680:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078152.689565:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078152.689574:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078152/real 1769078152] req@ffff9350103f8700 x1855008913591680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078152.689582:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913591680/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078152.689588:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078152.689590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078152.689593:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078152.689597:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078152.689607:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078152.689613:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078152/real 1769078152] req@ffff9350103fb800 x1855008913591808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078152.689620:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913591808/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078152.689631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078152.689633:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078152.689634:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078152.689656:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078152.689665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078152.689671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078152/real 1769078152] req@ffff9350103fb480 x1855008913591936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078152.689678:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913591936/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078152.689684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078152.689686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078152.689688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913591936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078152.689692:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592064:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078152.689701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078152.689706:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078152/real 1769078152] req@ffff9350103fad80 x1855008913592064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078152.689713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913592064/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078152.689718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078152.689720:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078152.689723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078152.689727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078152.689757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078152.689765:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078152/real 1769078152] req@ffff9350103f8380 x1855008913592192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078152.689773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913592192/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078152.689778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078152.689781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078152.689783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078152.689787:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078152.689796:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078152.689802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078152/real 1769078152] req@ffff9350103fb100 x1855008913592320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078152.689809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913592320/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078152.689814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078152.689816:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078152.689818:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078152.881183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078153.393164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078153.393164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078153.393168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078153.393341:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078153.713189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078153.905166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078154.417203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078154.417203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078154.417203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078154.417402:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078154.737209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078154.929135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078155.441145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078155.441145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078155.441179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078155.441200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078155.761226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078155.953214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078156.465178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078156.465178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078156.465196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078156.465212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078156.785192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078156.977154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078157.489161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078157.489161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078157.489162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078157.489221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078157.809222:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078157.809233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1769078157.809236:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078157.809237:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078157.809247:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3863 00000100:00080000:3.0:1769078157.809257:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078157.809263:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078157.809301:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913592448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078157.809334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078157.809339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078157.809341:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078157.809345:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3863 00000100:00080000:3.0:1769078157.809350:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078157.809354:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078157.809364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078157.809378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913592576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078157.809386:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078157.809390:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913592448:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078157.809390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078157.809393:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078157.809400:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3863 00000100:00080000:3.0:1769078157.809403:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078157.809405:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078157.809409:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078157.809418:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913592704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078157.809424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078157.809426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078157.809428:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078157.809430:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3863 00000100:00080000:3.0:1769078157.809431:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078157.809433:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078157.809437:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078157.809442:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913592832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078157.809447:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078157.809449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078157.809451:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078157.809454:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3863 00000100:00080000:3.0:1769078157.809456:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078157.809461:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078157.809463:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769078157.809465:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078157.809474:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913592960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078157.809481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078157.809483:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078157.809484:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078157.809486:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3863 00000100:00080000:3.0:1769078157.809488:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078157.809491:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078157.809493:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078157.809504:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078157/real 1769078157] req@ffff935002a7fb80 x1855008913592448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078157.809504:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913593088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078157.809527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078157.809530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078157.809532:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078157.809535:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3863 00000100:00080000:3.0:1769078157.809537:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078157.809538:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913592448/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078157.809541:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078157.809546:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078157.809552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078157.809556:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913593216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078157.809559:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078157.809566:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913592448:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078157.809590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913592576:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078157.809605:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078157.809613:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078157/real 1769078157] req@ffff935002a7df80 x1855008913592576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078157.809621:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913592576/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078157.809626:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078157.809628:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078157.809630:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913592576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078157.809634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913592704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078157.809667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078157.809675:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078157/real 1769078157] req@ffff935002a7f800 x1855008913592704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078157.809683:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913592704/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078157.809692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078157.809693:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078157.809695:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913592704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078157.809699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913592832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078157.809711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078157.809716:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078157/real 1769078157] req@ffff935002a7c000 x1855008913592832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078157.809722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913592832/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078157.809727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078157.809728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078157.809730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913592832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078157.809733:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592960:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078157.809741:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078157.809746:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078157/real 1769078157] req@ffff935002a7c700 x1855008913592960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078157.809752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913592960/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078157.809756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078157.809758:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078157.809760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913592960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078157.809763:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078157.809771:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078157.809776:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078157/real 1769078157] req@ffff935002a7ca80 x1855008913593088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078157.809782:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913593088/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078157.809786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078157.809788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078157.809789:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078157.809792:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078157.809801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078157.809805:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078157/real 1769078157] req@ffff935002a7f100 x1855008913593216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078157.809811:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913593216/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078157.809816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078157.809817:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078157.809819:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078158.001171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078158.513163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078158.513163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078158.513182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078158.513227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078158.833183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078159.025165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078159.537118:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078159.537119:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078159.537119:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078159.537166:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078159.857136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078160.049139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078160.561147:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078160.561147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078160.561147:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078160.561173:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078160.881185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078161.073170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078161.585180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078161.585180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078161.585186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078161.585226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078161.905170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078162.097194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078162.609138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078162.609138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078162.609138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078162.609169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078162.929112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.929122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078162.929127:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.929137:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3868 00000100:00100000:0.0:1769078162.929138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078162.929150:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.929158:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078162.929194:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913593344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.929215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.929217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078162.929219:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.929222:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3868 00000100:00080000:3.0:1769078162.929225:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.929229:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078162.929239:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078162.929253:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913593344:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078162.929254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913593472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.929270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.929274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078162.929276:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.929280:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3868 00000100:00080000:3.0:1769078162.929283:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.929286:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078162.929290:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078162.929299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913593600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.929306:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.929308:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078162.929309:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.929311:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3868 00000100:00080000:3.0:1769078162.929313:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.929315:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078162.929318:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078162.929324:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078162.929327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913593728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.929334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.929336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078162.929338:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.929340:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3868 00000100:00080000:3.0:1769078162.929343:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.929345:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078162.929349:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078162.929359:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913593856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078162.929361:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff935002a7dc00 x1855008913593344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078162.929372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.929377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078162.929378:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.929381:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3868 00000100:00100000:2.0:1769078162.929384:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913593344/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078162.929384:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.929389:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078162.929395:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078162.929399:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078162.929404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913593984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078162.929405:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078162.929411:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913593344:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078162.929416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.929419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078162.929420:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.929423:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3868 00000100:00080000:3.0:1769078162.929425:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.929427:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078162.929430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078162.929434:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913593472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078162.929438:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913594112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078162.929456:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078162.929465:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff935002a7d180 x1855008913593472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.929472:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913593472/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.929477:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.929479:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078162.929481:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913593472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078162.929485:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913593600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078162.929494:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078162.929499:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff935002a7e300 x1855008913593600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.929506:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913593600/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.929510:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.929512:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078162.929513:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913593600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078162.929516:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913593728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078162.929524:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078162.929529:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff935002a7c380 x1855008913593728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.929535:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913593728/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.929540:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.929541:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078162.929543:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913593728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078162.929546:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593856:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078162.929554:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078162.929572:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff935002a7f100 x1855008913593856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.929578:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913593856/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.929584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.929585:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078162.929587:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078162.929590:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078162.929600:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078162.929605:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff935002a7ca80 x1855008913593984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.929611:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913593984/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.929616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.929617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078162.929619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913593984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078162.929631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913594112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078162.929640:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078162.929645:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff935002a7c700 x1855008913594112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.929651:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913594112/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.929656:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.929658:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078162.929660:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913594112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078163.121181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078163.633140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078163.633157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078163.633158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078163.633227:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078163.953187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078164.145215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078164.657159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078164.657159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078164.657187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078164.657199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078164.977207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078165.169151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078165.681181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078165.681220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078165.681237:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078165.681267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078166.001205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078166.193160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078166.705178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078166.705199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078166.705200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078166.705216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078167.025203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078167.217159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078167.729142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078167.729149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078167.729183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078167.729221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078168.049169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078168.049180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078168.049185:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078168.049196:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3873 00000100:00080000:0.0:1769078168.049215:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078168.049223:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078168.049263:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913594240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078168.049294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078168.049298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078168.049300:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078168.049304:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3873 00000100:00080000:0.0:1769078168.049308:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078168.049311:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078168.049320:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078168.049336:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913594368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078168.049346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078168.049349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078168.049351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078168.049368:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913594240:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078168.049369:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3873 00000100:00080000:0.0:1769078168.049374:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078168.049387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078168.049394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078168.049412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913594496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078168.049428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078168.049431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078168.049433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078168.049438:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3873 00000100:00080000:0.0:1769078168.049442:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078168.049445:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078168.049450:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078168.049464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913594624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078168.049488:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078168.049497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078168.049504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078168.049507:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078168.049516:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3873 00000100:00080000:0.0:1769078168.049523:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078168.049527:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078168.049533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078168.049551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913594752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078168.049555:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078168/real 1769078168] req@ffff9350103f9180 x1855008913594240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078168.049597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078168.049602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078168.049605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078168.049612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913594240/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078168.049616:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3873 00000100:00080000:0.0:1769078168.049628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078168.049642:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078168.049654:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769078168.049655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078168.049669:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078168.049677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913594880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078168.049681:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913594240:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078168.049703:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078168.049706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078168.049708:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078168.049710:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3873 00000100:00080000:0.0:1769078168.049713:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078168.049715:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078168.049718:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078168.049732:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913595008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078168.049732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913594368:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078168.049786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078168.049793:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769078168.049810:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078168/real 1769078168] req@ffff9350103f9c00 x1855008913594368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078168.049832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913594368/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078168.049844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078168.049849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078168.049853:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913594368:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078168.049862:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913594496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078168.049887:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078168.049900:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078168/real 1769078168] req@ffff9350103fbb80 x1855008913594496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078168.049915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913594496/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078168.049930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078168.049933:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078168.049936:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913594496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078168.049941:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913594624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078168.049958:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078168.049966:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078168/real 1769078168] req@ffff9350103f9500 x1855008913594624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078168.049974:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913594624/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078168.049979:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078168.049982:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078168.049984:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913594624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078168.049988:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913594752:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078168.049999:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078168.050004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078168/real 1769078168] req@ffff9350103f9f80 x1855008913594752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078168.050012:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913594752/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078168.050018:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078168.050020:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078168.050022:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913594752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078168.050026:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913594880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078168.050036:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078168.050041:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078168/real 1769078168] req@ffff9350103fb100 x1855008913594880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078168.050081:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913594880/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078168.050087:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078168.050090:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078168.050092:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913594880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078168.050111:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078168.050124:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078168.050131:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078168/real 1769078168] req@ffff9350103f8380 x1855008913595008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078168.050139:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913595008/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078168.050149:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078168.050151:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078168.050153:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078168.241133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078168.753168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078168.753176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078168.753200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078168.753279:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078169.073235:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078169.265135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078169.777188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078169.777188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078169.777195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078169.777265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078170.097228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078170.289190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078170.801157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078170.801157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078170.801157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078170.801195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078171.121186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078171.313162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078171.825163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078171.825163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078171.825209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078171.825277:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078172.145125:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078172.337155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078172.849136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078172.849136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078172.849185:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078172.849228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078173.169213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078173.169219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078173.169225:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078173.169231:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078173.169241:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3878 00000100:00080000:3.0:1769078173.169253:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078173.169260:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078173.169303:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913595136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078173.169328:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078173.169331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078173.169332:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078173.169335:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3878 00000100:00080000:3.0:1769078173.169338:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078173.169340:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078173.169350:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078173.169359:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913595264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078173.169366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078173.169368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078173.169369:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078173.169372:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3878 00000100:00080000:3.0:1769078173.169374:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078173.169376:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078173.169379:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078173.169400:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913595392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078173.169407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078173.169409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078173.169411:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078173.169413:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3878 00000100:00080000:3.0:1769078173.169416:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078173.169419:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078173.169423:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078173.169430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913595520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078173.169438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078173.169441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078173.169442:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078173.169445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3878 00000100:00080000:3.0:1769078173.169447:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078173.169450:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078173.169453:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078173.169461:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913595648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078173.169468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078173.169471:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078173.169472:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078173.169476:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3878 00000100:00080000:3.0:1769078173.169478:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078173.169486:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169489:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913595136:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769078173.169490:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078173.169503:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913595776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078173.169510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078173.169512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078173.169513:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078173.169516:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3878 00000100:00080000:3.0:1769078173.169518:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078173.169521:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078173.169524:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078173.169532:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913595904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078173.169588:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169628:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078173/real 1769078173] req@ffff935002a7c000 x1855008913595136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078173.169644:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913595136/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078173.169653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078173.169657:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078173.169660:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913595136:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078173.169666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913595264:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078173.169683:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078173/real 1769078173] req@ffff935002a7f800 x1855008913595264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078173.169700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913595264/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078173.169718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078173.169721:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078173.169724:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913595264:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078173.169728:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913595392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078173.169741:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169748:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078173/real 1769078173] req@ffff935002a7df80 x1855008913595392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078173.169756:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913595392/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078173.169762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078173.169764:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078173.169766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913595392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078173.169770:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913595520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078173.169779:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078173/real 1769078173] req@ffff935002a7fb80 x1855008913595520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078173.169792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913595520/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078173.169797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078173.169799:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078173.169802:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913595520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078173.169826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595648:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078173.169837:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169843:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078173/real 1769078173] req@ffff935002a7c700 x1855008913595648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078173.169850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913595648/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078173.169855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078173.169858:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078173.169867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078173.169871:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078173.169882:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169888:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078173/real 1769078173] req@ffff935002a7ca80 x1855008913595776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078173.169896:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913595776/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078173.169901:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078173.169903:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078173.169905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078173.169909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078173.169917:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078173.169923:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078173/real 1769078173] req@ffff935002a7f100 x1855008913595904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078173.169930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913595904/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078173.169935:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078173.169937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078173.169939:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913595904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078173.361217:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078173.873218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078173.873219:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078173.873219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078173.873263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078174.193150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078174.385150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078174.897170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078174.897170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078174.897231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078174.897245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078175.217201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078175.409199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078175.921152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078175.921152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078175.921214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078175.921221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078176.241180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078176.433136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078176.945193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078176.945193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078176.945193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078176.945269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078177.265131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078177.457202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078177.969183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078177.969185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078177.969192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078177.969260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078178.289138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078178.289138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078178.289149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078178.289154:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078178.289165:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3883 00000100:00080000:3.0:1769078178.289177:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078178.289185:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078178.289224:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913596032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078178.289256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078178.289260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078178.289263:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078178.289268:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3883 00000100:00080000:3.0:1769078178.289272:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078178.289277:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078178.289288:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078178.289315:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913596032:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078178.289320:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913596160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078178.289337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078178.289341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078178.289344:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078178.289349:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3883 00000100:00080000:3.0:1769078178.289354:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078178.289359:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078178.289365:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078178.289379:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913596288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078178.289385:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078178.289397:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078178.289402:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078178.289405:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078178.289410:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3883 00000100:00080000:3.0:1769078178.289416:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078178.289420:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078178.289424:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078178/real 1769078178] req@ffff935002a7c380 x1855008913596032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078178.289428:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078178.289444:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913596032/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078178.289450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913596416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078178.289455:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078178.289464:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078178.289470:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913596032:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078178.289471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078178.289478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078178.289480:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078178.289485:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3883 00000100:00080000:3.0:1769078178.289490:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078178.289492:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913596160:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078178.289499:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078178.289507:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078178.289512:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078178.289527:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078178/real 1769078178] req@ffff935002a7e300 x1855008913596160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078178.289527:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913596544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078178.289564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078178.289569:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913596160/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078178.289573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078178.289578:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078178.289582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078178.289587:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3883 00000100:00080000:2.0:1769078178.289589:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078178.289594:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913596160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078178.289598:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913596288:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769078178.289605:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078178.289613:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078178.289615:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769078178.289625:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078178.289626:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078178/real 1769078178] req@ffff935002a7d180 x1855008913596288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078178.289638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913596288/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078178.289644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078178.289648:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913596672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078178.289649:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078178.289656:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913596288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078178.289666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913596416:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769078178.289675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078178.289681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769078178.289682:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078178.289691:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078178/real 1769078178] req@ffff935002a7dc00 x1855008913596416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078178.289696:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078178.289700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913596416/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078178.289706:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3883 00000100:00080000:3.0:1769078178.289715:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078178.289718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078178.289724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078178.289725:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078178.289728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913596416:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769078178.289736:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078178.289743:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913596544:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078178.289759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078178.289763:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913596800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078178.289770:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078178/real 1769078178] req@ffff935002a7c380 x1855008913596544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078178.289792:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913596544/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078178.289799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078178.289802:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078178.289804:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913596544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078178.289809:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913596672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078178.289819:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078178.289824:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078178/real 1769078178] req@ffff935002a7f100 x1855008913596672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078178.289832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913596672/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078178.289837:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078178.289838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078178.289841:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913596672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078178.289849:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913596800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078178.289858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078178.289863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078178/real 1769078178] req@ffff935002a7ca80 x1855008913596800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078178.289870:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913596800/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078178.289879:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078178.289881:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078178.289885:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913596800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078178.481139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078178.993187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078178.993194:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078178.993194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078178.993230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078179.313309:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078179.505179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078180.017171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078180.017171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078180.017182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078180.017317:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078180.337142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078180.529187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078181.041155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078181.041156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078181.041156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078181.041262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078181.361127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078181.553163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078182.065158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078182.065158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078182.065203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078182.065278:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078182.385178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078182.577144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078183.089142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078183.089142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078183.089142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078183.089242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078183.409154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078183.409159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078183.409167:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078183.409171:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078183.409181:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3889 00000100:00080000:3.0:1769078183.409192:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078183.409198:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078183.409254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913596928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078183.409280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078183.409283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078183.409285:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078183.409287:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3889 00000100:00080000:3.0:1769078183.409290:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078183.409292:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078183.409301:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078183.409309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913597056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078183.409316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078183.409319:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078183.409321:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078183.409323:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3889 00000100:00080000:3.0:1769078183.409325:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078183.409327:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078183.409331:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078183.409334:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913596928:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078183.409339:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913597184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078183.409346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078183.409348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078183.409349:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078183.409351:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3889 00000100:00080000:3.0:1769078183.409353:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078183.409355:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078183.409358:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078183.409364:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913597312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078183.409370:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078183.409372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078183.409373:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078183.409375:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3889 00000100:00080000:3.0:1769078183.409393:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078183.409396:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078183.409399:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078183.409406:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913597440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078183.409411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078183.409413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078183.409414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078183.409417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3889 00000100:00080000:3.0:1769078183.409419:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078183.409421:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078183.409423:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078183.409434:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913597568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078183.409440:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597056:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078183.409441:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078183.409445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078183.409447:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078183.409452:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3889 00000100:00080000:3.0:1769078183.409454:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078183.409456:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078183.409460:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078183.409468:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913597696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078183.409492:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078183.409516:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078183.409540:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913597440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078183.409564:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913597568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078183.409582:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913597696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078183.601122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078184.113156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078184.113156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078184.113161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078184.113254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078184.433241:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078184.625162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078185.137126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078185.137128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078185.137131:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078185.137179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078185.457268:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078185.649203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078186.161200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078186.161203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078186.161203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078186.161247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078186.481407:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:1.0:1769078186.481486:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769078186.481544:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769078186.481572:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078186.481634:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078186.481644:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078186.481652:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078186.481660:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078186.481669:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078186.481676:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078186.481778:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078183/real 1769078186] req@ffff935002a7df80 x1855008913597184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078186.481809:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913597184/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078186.481823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078186.481829:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078186.481835:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078186.481843:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078183/real 1769078186] req@ffff935002a7f800 x1855008913597312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078186.481855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913597312/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078186.481863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078186.481866:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078186.481869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078186.481876:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078183/real 1769078186] req@ffff935002a7c000 x1855008913597440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078186.481887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913597440/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078186.481895:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078186.481898:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078186.481901:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913597440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078186.481907:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078183/real 1769078186] req@ffff935002a7ca80 x1855008913597568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078186.481918:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913597568/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078186.481925:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078186.481928:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078186.481930:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913597568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078186.481937:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078183/real 1769078186] req@ffff935002a7f100 x1855008913597696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078186.481947:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913597696/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078186.481953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078186.481956:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078186.481959:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913597696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078186.481966:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078183/real 1769078186] req@ffff935002a7c700 x1855008913596928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078186.481977:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913596928/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078186.481985:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078186.481989:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078186.481992:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913596928:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078186.481999:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078183/real 1769078186] req@ffff935002a7fb80 x1855008913597056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078186.482010:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913597056/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078186.482021:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078186.482025:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078186.482029:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769078186.673191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078187.185161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078187.185161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078187.185162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078187.185275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078187.505136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078187.697137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078188.209158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078188.209159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078188.209159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078188.209208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078188.529196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078188.529208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078188.529212:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078188.529223:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3894 00000100:00080000:3.0:1769078188.529236:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078188.529243:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078188.529273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913597824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078188.529294:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078188.529297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078188.529298:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078188.529302:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3894 00000100:00080000:3.0:1769078188.529305:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078188.529308:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078188.529317:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078188.529326:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913597952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078188.529334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078188.529340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769078188.529346:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913597824:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078188.529347:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078188.529372:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3894 00000100:00080000:3.0:1769078188.529376:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078188.529380:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078188.529392:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078188.529409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913598080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078188.529419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078188.529423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078188.529425:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078188.529430:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3894 00000100:00080000:3.0:1769078188.529434:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078188.529439:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078188.529444:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078188.529455:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913598208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078188.529464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078188.529470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1769078188.529476:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078188.529478:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078188.529486:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3894 00000100:00080000:3.0:1769078188.529490:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078188.529494:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078188.529499:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078188.529517:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913598336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078188.529528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078188.529532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078188.529535:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078188.529541:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3894 00000100:00080000:3.0:1769078188.529546:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078188.529551:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078188.529556:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078188/real 1769078188] req@ffff935002a7c380 x1855008913597824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078188.529560:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078188.529588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913598464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078188.529599:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913597824/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078188.529612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078188.529618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078188.529621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078188.529622:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078188.529642:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078188.529648:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3894 00000100:00100000:2.0:1769078188.529659:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913597824:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078188.529661:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078188.529674:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078188.529682:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078188.529695:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913598592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078188.529710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078188.529756:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078188.529775:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078188/real 1769078188] req@ffff935002a7dc00 x1855008913597952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078188.529790:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913597952/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078188.529807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078188.529812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078188.529816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913597952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078188.529826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078188.529835:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000800:00000100:2.0:1769078188.529855:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078188.529872:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078188/real 1769078188] req@ffff935002a7d180 x1855008913598080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078188.529887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913598080/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078188.529898:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078188.529903:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078188.529907:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078188.529914:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078188.529932:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078188.529943:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078188/real 1769078188] req@ffff935002a7e300 x1855008913598208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078188.529955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913598208/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078188.529963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078188.529966:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078188.529971:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078188.529978:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913598336:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078188.529997:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078188.530010:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078188/real 1769078188] req@ffff935002a7fb80 x1855008913598336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078188.530021:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913598336/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078188.530029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078188.530032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078188.530035:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913598336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078188.530042:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913598464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078188.530110:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078188.530122:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078188/real 1769078188] req@ffff935002a7c700 x1855008913598464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078188.530149:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913598464/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078188.530156:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078188.530159:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078188.530162:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913598464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078188.530187:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913598592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078188.530209:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078188.530222:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078188/real 1769078188] req@ffff935002a7f100 x1855008913598592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078188.530233:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913598592/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078188.530240:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078188.530243:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078188.530246:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913598592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078188.721163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078189.233174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078189.233174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078189.233179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078189.233213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078189.553159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078189.745164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078190.257157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078190.257157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078190.257276:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078190.257336:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078190.577204:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078190.769210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078191.281197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078191.281198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078191.281198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078191.281233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078191.601160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078191.793179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078192.305150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078192.305150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078192.305150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078192.305202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078192.625198:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078192.817181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078193.329212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078193.329212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078193.329221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078193.329253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078193.649142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078193.649175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078193.649184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078193.649188:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078193.649197:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3899 00000100:00080000:0.0:1769078193.649209:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078193.649216:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078193.649253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913598720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078193.649274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078193.649277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078193.649279:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078193.649287:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3899 00000100:00080000:0.0:1769078193.649292:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078193.649297:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649308:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913598720:192.168.204.121@tcp:250:kworker.0 00000008:00100000:0.0:1769078193.649311:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078193.649324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913598848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078193.649332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078193.649335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078193.649336:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078193.649339:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3899 00000100:00080000:0.0:1769078193.649342:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078193.649344:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078193.649348:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078193.649354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913598976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078193.649360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078193.649362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078193.649364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078193.649366:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3899 00000100:00080000:0.0:1769078193.649368:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078193.649370:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078193.649373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078193.649378:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078193.649380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913599104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078193.649387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078193.649390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078193.649391:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078193.649393:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3899 00000100:00080000:0.0:1769078193.649395:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078193.649398:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078193.649401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078193.649406:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913599232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078193.649411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078193.649413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078193.649414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078193.649417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3899 00000100:00080000:0.0:1769078193.649419:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078193.649421:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078193.649424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078193.649431:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078193/real 1769078193] req@ffff9350103fad80 x1855008913598720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078193.649433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913599360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078193.649458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078193.649460:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078193.649461:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078193.649464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3899 00000100:00080000:0.0:1769078193.649466:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078193.649469:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649472:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913598720/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769078193.649474:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078193.649485:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769078193.649487:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913599488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078193.649491:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078193.649497:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913598720:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078193.649522:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598848:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078193.649539:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649547:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078193/real 1769078193] req@ffff9350103fb480 x1855008913598848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078193.649555:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913598848/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078193.649565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078193.649567:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078193.649570:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078193.649573:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078193.649583:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649588:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078193/real 1769078193] req@ffff9350103fb800 x1855008913598976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078193.649595:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913598976/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078193.649600:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078193.649601:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078193.649603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913598976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078193.649606:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913599104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078193.649615:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649620:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078193/real 1769078193] req@ffff9350103f8700 x1855008913599104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078193.649627:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913599104/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078193.649631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078193.649632:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078193.649635:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913599104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078193.649638:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913599232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078193.649647:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649651:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078193/real 1769078193] req@ffff9350103faa00 x1855008913599232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078193.649657:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913599232/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078193.649662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078193.649664:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078193.649665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913599232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078193.649668:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913599360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078193.649677:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649681:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078193/real 1769078193] req@ffff9350103f8380 x1855008913599360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078193.649688:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913599360/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078193.649692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078193.649694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078193.649697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913599360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078193.649702:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913599488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078193.649717:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078193.649728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078193/real 1769078193] req@ffff9350103fb100 x1855008913599488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078193.649740:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913599488/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078193.649748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078193.649751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078193.649754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913599488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078193.841114:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078194.353150:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078194.353151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078194.353158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078194.353186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078194.673205:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078194.865180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078195.377113:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078195.377147:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078195.377161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078195.377256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078195.697203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078195.889117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078196.401173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078196.401173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078196.401180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078196.401208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078196.721190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078196.913195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078197.425170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078197.425170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078197.425176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078197.425205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078197.745132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078197.937167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078198.449162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078198.449162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078198.449162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078198.449195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078198.769198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078198.769199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078198.769210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078198.769215:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078198.769228:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3904 00000100:00080000:3.0:1769078198.769240:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078198.769249:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078198.769288:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913599616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078198.769313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078198.769316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078198.769317:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078198.769319:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3904 00000100:00080000:3.0:1769078198.769322:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078198.769325:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078198.769334:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078198.769343:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913599744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078198.769349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078198.769352:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078198.769354:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078198.769356:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3904 00000100:00080000:3.0:1769078198.769358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078198.769360:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078198.769364:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078198.769367:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913599616:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078198.769372:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913599872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078198.769381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078198.769383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078198.769384:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078198.769386:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3904 00000100:00080000:3.0:1769078198.769388:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078198.769390:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078198.769393:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078198.769399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913600000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078198.769404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078198.769406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078198.769407:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078198.769409:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3904 00000100:00080000:3.0:1769078198.769411:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078198.769413:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078198.769416:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078198.769425:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913600128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078198.769431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078198.769433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078198.769434:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078198.769437:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3904 00000100:00080000:3.0:1769078198.769450:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078198.769453:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078198.769458:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078198.769463:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078198.769471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913600256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078198.769482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078198.769485:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078198.769487:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078198.769491:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3904 00000100:00080000:3.0:1769078198.769494:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078198.769498:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078198.769503:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078198.769514:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078198/real 1769078198] req@ffff935002a7ca80 x1855008913599616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078198.769517:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913600384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078198.769553:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913599616/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078198.769565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078198.769570:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078198.769574:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913599616:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078198.769582:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913599744:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078198.769601:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078198.769613:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078198/real 1769078198] req@ffff935002a7c000 x1855008913599744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078198.769624:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913599744/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078198.769633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078198.769637:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078198.769640:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913599744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078198.769666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913599872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078198.769685:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078198.769695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078198/real 1769078198] req@ffff935002a7f800 x1855008913599872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078198.769707:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913599872/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078198.769715:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078198.769718:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078198.769722:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913599872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078198.769727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078198.769744:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078198.769754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078198/real 1769078198] req@ffff935002a7df80 x1855008913600000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078198.769765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913600000/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078198.769773:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078198.769776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078198.769780:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078198.769785:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913600128:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078198.769803:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078198.769813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078198/real 1769078198] req@ffff935002a7f100 x1855008913600128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078198.769824:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913600128/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078198.769832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078198.769835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078198.769838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913600128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078198.769843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913600256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078198.769860:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078198.769870:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078198/real 1769078198] req@ffff935002a7c700 x1855008913600256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078198.769881:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913600256/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078198.769888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078198.769891:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078198.769894:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913600256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078198.769900:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913600384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078198.769917:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078198.769927:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078198/real 1769078198] req@ffff935002a7fb80 x1855008913600384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078198.769938:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913600384/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078198.769945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078198.769948:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078198.769951:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913600384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078198.961156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078199.473175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078199.473175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078199.473175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078199.473209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078199.793127:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078199.985212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078200.497172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078200.497172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078200.497173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078200.497207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078200.817217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078201.009158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078201.521173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078201.521173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078201.521173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078201.521208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078201.841126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078202.033223:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078202.545137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078202.545138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078202.545138:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078202.545192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078202.865179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078203.057152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078203.569208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078203.569208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078203.569208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078203.569245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078203.889151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078203.889151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078203.889161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078203.889166:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078203.889174:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3909 00000100:00080000:3.0:1769078203.889185:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078203.889191:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078203.889226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913600512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078203.889253:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078203.889256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078203.889258:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078203.889261:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3909 00000100:00080000:3.0:1769078203.889264:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078203.889268:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078203.889276:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078203.889286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913600640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078203.889296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078203.889298:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078203.889300:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078203.889305:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3909 00000100:00100000:2.0:1769078203.889307:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913600512:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078203.889309:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078203.889316:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078203.889321:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078203.889330:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913600768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078203.889338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078203.889342:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078203.889343:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078203.889347:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3909 00000100:00080000:3.0:1769078203.889366:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078203.889369:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078203.889373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078203.889377:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078203.889386:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913600896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078203.889396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078203.889399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078203.889401:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078203.889404:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3909 00000100:00080000:3.0:1769078203.889407:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078203.889410:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078203.889414:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078203.889423:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff935002a7e300 x1855008913600512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078203.889427:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913601024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078203.889446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078203.889449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769078203.889452:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913600512/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078203.889453:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078203.889461:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3909 00000100:00080000:3.0:1769078203.889468:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078203.889470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078203.889474:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078203.889477:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769078203.889480:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078203.889481:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913600512:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078203.889496:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078203.889498:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913601152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078203.889507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078203.889510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078203.889512:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078203.889518:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078203.889518:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3909 00000100:00080000:3.0:1769078203.889525:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078203.889530:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078203.889533:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff935002a7d180 x1855008913600640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078203.889536:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078203.889546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913600640/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078203.889560:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913601280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078203.889564:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078203.889573:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078203.889578:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078203.889608:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078203.889624:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078203.889631:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff935002a7dc00 x1855008913600768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078203.889638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913600768/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078203.889643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078203.889652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078203.889655:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078203.889661:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078203.889677:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078203.889687:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff935002a7c380 x1855008913600896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078203.889698:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913600896/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078203.889706:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078203.889708:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078203.889710:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913600896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078203.889714:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601024:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078203.889726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078203.889733:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff935002a7fb80 x1855008913601024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078203.889739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913601024/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078203.889744:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078203.889746:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078203.889748:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601024:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078203.889751:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078203.889759:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078203.889764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff935002a7c700 x1855008913601152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078203.889770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913601152/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078203.889774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078203.889776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078203.889777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078203.889780:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078203.889787:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078203.889792:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff935002a7f100 x1855008913601280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078203.889799:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913601280/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078203.889803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078203.889805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078203.889806:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078204.081201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078204.593136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078204.593136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078204.593173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078204.593237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078204.913164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078205.105165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078205.617190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078205.617190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078205.617203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078205.617233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078205.937190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078206.129184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078206.641117:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078206.641118:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078206.641159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078206.641231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078206.961180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078207.153156:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078207.665135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078207.665149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078207.665180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078207.665181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078207.985174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078208.177193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078208.689139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078208.689139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078208.689166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078208.689174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078209.009145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078209.009176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078209.009184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078209.009192:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078209.009204:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3914 00000100:00080000:0.0:1769078209.009217:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078209.009225:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078209.009266:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913601408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078209.009294:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078209.009297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078209.009299:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078209.009302:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3914 00000100:00080000:0.0:1769078209.009304:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078209.009307:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078209.009317:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078209.009328:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913601536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078209.009338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078209.009338:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913601408:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078209.009345:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078209.009348:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078209.009352:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3914 00000100:00080000:0.0:1769078209.009354:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078209.009356:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078209.009360:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078209.009369:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913601664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078209.009381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078209.009383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078209.009385:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078209.009387:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3914 00000100:00080000:0.0:1769078209.009390:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078209.009392:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078209.009395:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078209.009402:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913601792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078209.009408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078209.009413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1769078209.009413:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078209.009418:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078209.009422:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3914 00000100:00080000:0.0:1769078209.009425:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078209.009428:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078209.009431:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078209.009439:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913601920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078209.009446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078209.009448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078209.009449:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078209.009453:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3914 00000100:00080000:0.0:1769078209.009456:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078209.009459:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078209/real 1769078209] req@ffff9350103f9f80 x1855008913601408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078209.009462:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078209.009470:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078209.009482:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913601408/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078209.009487:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913602048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078209.009495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078209.009503:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769078209.009505:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078209.009509:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913601408:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078209.009513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078209.009517:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078209.009522:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3914 00000100:00080000:0.0:1769078209.009526:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078209.009531:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078209.009534:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913601536:192.168.204.121@tcp:38:kworker.0 00000008:00100000:0.0:1769078209.009538:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078209.009555:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913602176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078209.009557:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078209.009569:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078209/real 1769078209] req@ffff9350103f9500 x1855008913601536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078209.009585:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913601536/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078209.009592:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078209.009594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078209.009597:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913601536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078209.009601:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913601664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078209.009614:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078209.009623:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078209/real 1769078209] req@ffff9350103fbb80 x1855008913601664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078209.009637:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913601664/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078209.009646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078209.009650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078209.009653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913601664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078209.009660:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913601792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078209.009678:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078209.009688:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078209/real 1769078209] req@ffff9350103f9c00 x1855008913601792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078209.009700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913601792/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078209.009714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078209.009717:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078209.009721:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913601792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078209.009726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601920:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078209.009742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078209.009750:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078209/real 1769078209] req@ffff9350103f9180 x1855008913601920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078209.009762:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913601920/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078209.009770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078209.009773:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078209.009776:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913601920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078209.009795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078209.009814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078209.009825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078209/real 1769078209] req@ffff9350103fb100 x1855008913602048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078209.009836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913602048/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078209.009843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078209.009847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078209.009860:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078209.009885:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078209.009904:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078209.009914:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078209/real 1769078209] req@ffff9350103f8380 x1855008913602176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078209.009926:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913602176/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078209.009933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078209.009936:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078209.009940:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078209.201164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078209.713174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078209.713174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078209.713174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078209.713248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078210.033212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078210.225214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078210.737161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078210.737161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078210.737161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078210.737194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078211.057128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078211.249133:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078211.761132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078211.761132:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078211.761132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078211.761179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078212.081200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078212.273194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078212.785168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078212.785168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078212.785168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078212.785234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078213.105194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078213.297125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078213.809160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078213.809160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078213.809160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078213.809195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078214.129172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078214.129182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1769078214.129184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078214.129187:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078214.129199:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3919 00000100:00080000:1.0:1769078214.129210:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078214.129216:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078214.129253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913602304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078214.129274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078214.129277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078214.129278:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078214.129281:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3919 00000100:00080000:1.0:1769078214.129283:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078214.129285:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078214.129294:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078214.129304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913602432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078214.129310:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078214.129312:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078214.129313:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078214.129316:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3919 00000100:00080000:1.0:1769078214.129317:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078214.129319:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078214.129322:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078214.129328:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913602560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078214.129334:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913602304:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078214.129336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078214.129342:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078214.129345:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078214.129349:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3919 00000100:00080000:1.0:1769078214.129352:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078214.129354:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078214.129356:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078214.129363:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913602688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078214.129368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078214.129370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078214.129371:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078214.129373:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3919 00000100:00080000:1.0:1769078214.129375:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078214.129377:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078214.129380:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078214.129390:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913602816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078214.129398:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078214.129401:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078214.129403:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078214.129407:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3919 00000100:00080000:1.0:1769078214.129411:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078214.129414:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078214.129419:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078214.129425:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769078214.129433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913602944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078214.129450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078214.129453:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078214.129455:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078214.129460:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3919 00000100:00080000:1.0:1769078214.129463:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078214.129466:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078214.129470:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078214.129480:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913603072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078214.129491:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078214/real 1769078214] req@ffff9350100cb800 x1855008913602304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078214.129511:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913602304/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078214.129520:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078214.129525:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078214.129529:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913602304:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078214.129536:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913602432:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078214.129559:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078214.129571:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078214/real 1769078214] req@ffff9350100ca300 x1855008913602432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078214.129579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913602432/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078214.129584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078214.129587:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078214.129589:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913602432:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078214.129609:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913602560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078214.129624:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078214.129631:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078214/real 1769078214] req@ffff9350100c8e00 x1855008913602560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078214.129638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913602560/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078214.129643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078214.129645:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078214.129647:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913602560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078214.129650:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913602688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078214.129659:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078214.129664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078214/real 1769078214] req@ffff9350100ca680 x1855008913602688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078214.129674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913602688/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078214.129679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078214.129680:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078214.129682:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913602688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078214.129685:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602816:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078214.129711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078214.129719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078214/real 1769078214] req@ffff9350100c8380 x1855008913602816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078214.129726:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913602816/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078214.129731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078214.129733:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078214.129735:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078214.129738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078214.129750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078214.129755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078214/real 1769078214] req@ffff9350100caa00 x1855008913602944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078214.129765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913602944/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078214.129777:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078214.129780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078214.129783:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913602944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078214.129789:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078214.129807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078214.129817:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078214/real 1769078214] req@ffff9350100c9500 x1855008913603072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078214.129828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913603072/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078214.129835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078214.129838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078214.129841:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078214.321163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078214.833165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078214.833165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078214.833166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078214.833197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078215.153123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078215.345202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078215.857163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078215.857163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078215.857163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078215.857201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078216.177158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078216.369142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078216.881161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078216.881161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078216.881172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078216.881293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078217.201139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078217.393149:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078217.905167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078217.905167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078217.905201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078217.905231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078218.225183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078218.417153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078218.929174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078218.929174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078218.929189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078218.929209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078219.249141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078219.249152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078219.249156:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078219.249166:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3924 00000100:00080000:1.0:1769078219.249177:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078219.249184:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078219.249224:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913603200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078219.249255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078219.249259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078219.249260:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078219.249264:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3924 00000100:00080000:1.0:1769078219.249268:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078219.249271:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078219.249280:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078219.249290:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913603328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078219.249296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078219.249299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078219.249300:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078219.249302:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3924 00000100:00080000:1.0:1769078219.249304:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078219.249306:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078219.249310:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078219.249319:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913603456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078219.249325:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078219.249327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078219.249328:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078219.249331:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3924 00000100:00080000:1.0:1769078219.249335:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078219.249337:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078219.249340:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078219.249346:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913603584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078219.249355:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913603200:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078219.249356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078219.249359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078219.249365:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078219.249375:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3924 00000100:00080000:1.0:1769078219.249377:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078219.249380:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078219.249383:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078219.249392:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913603712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078219.249400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078219.249403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078219.249405:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078219.249408:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3924 00000100:00080000:1.0:1769078219.249412:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078219.249416:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078219.249421:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078219.249433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913603840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078219.249443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078219.249447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078219.249452:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078219.249454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078219.249464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3924 00000100:00080000:1.0:1769078219.249469:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078219.249472:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078219.249476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078219.249486:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913603968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078219.249515:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078219/real 1769078219] req@ffff9350100c9880 x1855008913603200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078219.249536:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913603200/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078219.249549:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078219.249555:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078219.249561:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913603200:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078219.249572:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769078219.249573:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913603328:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078219.249604:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078219.249637:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078219/real 1769078219] req@ffff9350100cad80 x1855008913603328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078219.249650:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913603328/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078219.249659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078219.249663:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078219.249666:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913603328:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078219.249673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913603456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078219.249691:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078219.249699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078219/real 1769078219] req@ffff9350100c9c00 x1855008913603456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078219.249706:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913603456/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078219.249711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078219.249713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078219.249715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913603456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078219.249738:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913603584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078219.249748:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078219.249753:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078219/real 1769078219] req@ffff9350100cb100 x1855008913603584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078219.249760:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913603584/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078219.249764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078219.249766:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078219.249767:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913603584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078219.249771:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603712:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078219.249780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078219.249784:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078219/real 1769078219] req@ffff9350100c9180 x1855008913603712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078219.249791:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913603712/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078219.249795:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078219.249797:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078219.249798:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078219.249802:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078219.249809:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078219.249814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078219/real 1769078219] req@ffff9350100c8a80 x1855008913603840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078219.249820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913603840/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078219.249825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078219.249827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078219.249828:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078219.249832:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078219.249840:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078219.249844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078219/real 1769078219] req@ffff9350100c8000 x1855008913603968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078219.249850:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913603968/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078219.249853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078219.249864:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078219.249865:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913603968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078219.441172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078219.953140:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078219.953141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078219.953141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078219.953311:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078220.273166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078220.465148:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078220.977158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078220.977158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078220.977158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078220.977241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078221.297163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078221.489200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078222.001133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078222.001137:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078222.001163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078222.001203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078222.321185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078222.513129:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078223.025193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078223.025193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078223.025267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078223.025283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078223.345185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078223.537201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078224.049140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078224.049142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078224.049171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078224.049182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078224.369167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078224.369205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078224.369217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078224.369223:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078224.369235:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3930 00000100:00080000:3.0:1769078224.369249:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078224.369257:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078224.369304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913604096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078224.369339:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078224.369343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078224.369345:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078224.369350:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3930 00000100:00080000:3.0:1769078224.369354:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078224.369358:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078224.369369:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078224.369383:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913604224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078224.369394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078224.369397:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078224.369399:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078224.369405:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3930 00000100:00100000:2.0:1769078224.369408:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913604096:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078224.369409:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078224.369417:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078224.369424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078224.369436:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913604352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078224.369447:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078224.369451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078224.369453:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078224.369457:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3930 00000100:00080000:3.0:1769078224.369461:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078224.369464:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078224.369470:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078224.369481:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913604480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078224.369491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078224.369495:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078224.369497:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078224.369503:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078224.369503:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3930 00000100:00080000:3.0:1769078224.369510:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078224.369515:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078224.369520:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078224.369559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913604608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078224.369569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078224.369573:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078224/real 1769078224] req@ffff935002a7df80 x1855008913604096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078224.369574:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078224.369577:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078224.369585:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3930 00000100:00080000:3.0:1769078224.369593:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078224.369598:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078224.369604:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078224.369605:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913604096/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078224.369618:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913604736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078224.369631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078224.369634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078224.369639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078224.369640:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078224.369643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078224.369648:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913604096:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078224.369652:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3930 00000100:00080000:3.0:1769078224.369658:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078224.369662:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913604224:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078224.369665:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078224.369674:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078224.369687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913604864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078224.369697:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078224.369713:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078224/real 1769078224] req@ffff935002a7f800 x1855008913604224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078224.369727:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913604224/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078224.369736:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078224.369740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078224.369745:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913604224:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078224.369789:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913604352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078224.369810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078224.369822:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078224/real 1769078224] req@ffff935002a7c000 x1855008913604352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078224.369833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913604352/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078224.369841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078224.369844:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078224.369847:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913604352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078224.369854:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913604480:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078224.369873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078224.369883:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078224/real 1769078224] req@ffff935002a7ca80 x1855008913604480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078224.369893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913604480/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078224.369901:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078224.369904:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078224.369907:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913604480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078224.369913:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913604608:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078224.369931:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078224.369940:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078224/real 1769078224] req@ffff935002a7f100 x1855008913604608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078224.369951:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913604608/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078224.369957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078224.369960:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078224.369962:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913604608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078224.369967:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913604736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078224.369982:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078224.369990:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078224/real 1769078224] req@ffff935002a7c700 x1855008913604736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078224.370000:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913604736/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078224.370006:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078224.370009:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078224.370012:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913604736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078224.370016:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913604864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078224.370030:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078224.370039:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078224/real 1769078224] req@ffff935002a7fb80 x1855008913604864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078224.370082:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913604864/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078224.370090:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078224.370093:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078224.370096:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913604864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078224.561146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078225.073217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078225.073222:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078225.073250:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078225.073346:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078225.393176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078225.585180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078226.097188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078226.097191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078226.097197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078226.097246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078226.417249:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078226.609163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078227.121219:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078227.121225:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078227.121226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078227.121268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078227.441179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078227.633162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078228.145176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078228.145176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078228.145176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078228.145226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078228.465146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078228.657185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078229.169172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078229.169173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078229.169182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078229.169205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078229.489187:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078229.489194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078229.489205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078229.489210:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078229.489222:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3935 00000100:00080000:3.0:1769078229.489234:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078229.489243:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078229.489285:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913604992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078229.489315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078229.489318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078229.489320:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078229.489325:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3935 00000100:00080000:3.0:1769078229.489328:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078229.489332:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078229.489345:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078229.489356:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913605120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078229.489368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078229.489371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078229.489375:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078229.489376:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913604992:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078229.489381:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3935 00000100:00080000:3.0:1769078229.489388:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078229.489392:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078229.489397:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078229.489407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913605248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078229.489415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078229.489418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078229.489420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078229.489424:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3935 00000100:00080000:3.0:1769078229.489427:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078229.489431:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078229.489449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078229.489459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913605376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078229.489469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078229.489472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078229.489474:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078229.489478:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3935 00000100:00080000:3.0:1769078229.489482:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078229.489483:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078229.489489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078229.489494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078229.489507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913605504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078229.489516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078229.489518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078229.489520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078229.489524:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3935 00000100:00080000:3.0:1769078229.489528:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078229.489532:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078229.489537:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078229/real 1769078229] req@ffff935002a7c380 x1855008913604992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078229.489539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078229.489552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913605632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078229.489561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078229.489566:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913604992/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078229.489567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078229.489570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078229.489577:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3935 00000100:00080000:3.0:1769078229.489584:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078229.489588:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078229.489588:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078229.489595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078229.489605:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078229.489607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913605760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078229.489614:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913604992:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078229.489646:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913605120:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078229.489670:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078229.489682:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078229/real 1769078229] req@ffff935002a7dc00 x1855008913605120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078229.489695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913605120/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078229.489703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078229.489706:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078229.489709:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913605120:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078229.489715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913605248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078229.489734:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078229.489744:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078229/real 1769078229] req@ffff935002a7d180 x1855008913605248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078229.489755:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913605248/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078229.489762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078229.489765:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078229.489768:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913605248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078229.489774:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913605376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078229.489789:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078229.489799:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078229/real 1769078229] req@ffff935002a7e300 x1855008913605376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078229.489810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913605376/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078229.489817:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078229.489820:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078229.489823:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913605376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078229.489828:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913605504:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078229.489845:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078229.489854:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078229/real 1769078229] req@ffff935002a7fb80 x1855008913605504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078229.489865:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913605504/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078229.489873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078229.489875:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078229.489887:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913605504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078229.489897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913605632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078229.489915:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078229.489925:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078229/real 1769078229] req@ffff935002a7c700 x1855008913605632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078229.489936:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913605632/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078229.489944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078229.489947:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078229.489950:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913605632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078229.489955:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913605760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078229.489973:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078229.489982:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078229/real 1769078229] req@ffff935002a7f100 x1855008913605760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078229.489989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913605760/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078229.489995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078229.489997:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078229.489998:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913605760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078229.681173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078230.193211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078230.193211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078230.193238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078230.193570:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078230.513206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078230.705193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078231.217137:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078231.217155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078231.217254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078231.217297:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078231.537189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078231.729183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078232.241193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078232.241193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078232.241193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078232.241313:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078232.561170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078232.753145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078233.265175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078233.265175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078233.265175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078233.265237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078233.585207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078233.777134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078234.289174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078234.289174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078234.289174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078234.289208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078234.609184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078234.609196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078234.609201:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078234.609210:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3940 00000100:00080000:3.0:1769078234.609223:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078234.609231:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078234.609279:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913605888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078234.609308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078234.609312:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078234.609313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078234.609318:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3940 00000100:00080000:3.0:1769078234.609321:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078234.609326:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078234.609337:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078234.609360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913606016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078234.609372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078234.609377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769078234.609378:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913605888:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078234.609380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078234.609390:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3940 00000100:00080000:3.0:1769078234.609393:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078234.609397:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078234.609403:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078234.609416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913606144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078234.609426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078234.609429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078234.609432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078234.609436:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3940 00000100:00080000:3.0:1769078234.609440:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078234.609444:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078234.609448:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769078234.609450:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000008:00100000:3.0:1769078234.609451:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078234.609466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913606272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078234.609475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078234.609478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078234.609480:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078234.609484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3940 00000100:00080000:3.0:1769078234.609488:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078234.609492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078234.609497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078234.609500:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078234/real 1769078234] req@ffff935002a7ca80 x1855008913605888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078234.609511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913606400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078234.609523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078234.609527:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913605888/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078234.609528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078234.609532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078234.609540:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3940 00000100:00080000:2.0:1769078234.609543:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078234.609544:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078234.609549:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078234.609550:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078234.609555:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913605888:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769078234.609557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078234.609568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913606528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078234.609578:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606016:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078234.609579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078234.609585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078234.609587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078234.609592:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3940 00000100:00080000:3.0:1769078234.609597:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078234.609599:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078234.609603:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078234.609609:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078234.609618:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078234/real 1769078234] req@ffff935002a7c000 x1855008913606016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078234.609621:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913606656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078234.609653:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913606016/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078234.609660:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078234.609663:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078234.609665:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606016:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078234.609670:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078234.609682:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078234.609688:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078234/real 1769078234] req@ffff935002a7f800 x1855008913606144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078234.609695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913606144/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078234.609704:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078234.609706:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078234.609708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078234.609711:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078234.609720:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078234.609725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078234/real 1769078234] req@ffff935002a7df80 x1855008913606272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078234.609737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913606272/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078234.609742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078234.609744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078234.609746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078234.609749:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913606400:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078234.609758:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078234.609763:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078234/real 1769078234] req@ffff935002a7f100 x1855008913606400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078234.609770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913606400/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078234.609775:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078234.609777:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078234.609779:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913606400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078234.609792:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913606528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078234.609801:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078234.609806:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078234/real 1769078234] req@ffff935002a7c700 x1855008913606528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078234.609813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913606528/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078234.609817:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078234.609819:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078234.609821:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913606528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078234.609824:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913606656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078234.609832:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078234.609837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078234/real 1769078234] req@ffff935002a7fb80 x1855008913606656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078234.609843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913606656/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078234.609848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078234.609849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078234.609851:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913606656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078234.801130:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078235.313197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078235.313197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078235.313201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078235.313238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078235.633286:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078235.825214:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078236.337169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078236.337172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078236.337204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078236.337332:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078236.657141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078236.849181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078237.361176:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078237.361176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078237.361316:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078237.361372:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078237.681169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078237.873138:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078238.385170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078238.385171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078238.385196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078238.385230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078238.705181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078238.897176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078239.409165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078239.409165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078239.409190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078239.409311:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769078239.729210:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769078239.729212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078239.729223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078239.729228:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078239.729239:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3945 00000100:00080000:1.0:1769078239.729253:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078239.729260:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078239.729316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913606784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078239.729343:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078239.729347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078239.729349:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078239.729353:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3945 00000100:00080000:1.0:1769078239.729356:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078239.729361:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078239.729373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078239.729386:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913606784:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078239.729389:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913606912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078239.729401:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078239.729405:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078239.729407:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078239.729410:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3945 00000100:00080000:1.0:1769078239.729413:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078239.729416:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078239.729421:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078239.729432:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913607040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078239.729442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078239.729445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078239.729447:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078239.729452:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078239.729453:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3945 00000100:00080000:1.0:1769078239.729458:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078239.729462:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078239.729467:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078239.729479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913607168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078239.729487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078239.729493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1769078239.729493:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078239/real 1769078239] req@ffff9350100c9f80 x1855008913606784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078239.729496:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078239.729505:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3945 00000100:00080000:1.0:1769078239.729509:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078239.729513:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078239.729517:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913606784/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769078239.729520:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078239.729534:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769078239.729536:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913607296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078239.729542:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078239.729551:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078239.729551:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913606784:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078239.729559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078239.729562:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078239.729566:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3945 00000100:00080000:1.0:1769078239.729570:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078239.729573:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078239.729577:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078239.729583:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769078239.729589:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913607424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078239.729600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078239.729603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078239.729605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078239.729610:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078239.729613:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3945 00000100:00080000:1.0:1769078239.729618:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078239.729622:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078239.729627:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078239.729627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078239/real 1769078239] req@ffff9350100c8000 x1855008913606912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078239.729642:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913607552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078239.729645:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913606912/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078239.729665:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078239.729667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078239.729670:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913606912:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078239.729678:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078239.729692:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078239.729699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078239/real 1769078239] req@ffff9350100c8a80 x1855008913607040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078239.729706:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913607040/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078239.729711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078239.729713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078239.729715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078239.729718:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078239.729727:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078239.729731:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078239/real 1769078239] req@ffff9350100c9180 x1855008913607168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078239.729738:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913607168/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078239.729742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078239.729744:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078239.729746:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078239.729748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913607296:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078239.729757:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078239.729762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078239/real 1769078239] req@ffff9350100cb100 x1855008913607296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078239.729769:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913607296/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078239.729774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078239.729775:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078239.729777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913607296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078239.729787:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913607424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078239.729796:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078239.729801:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078239/real 1769078239] req@ffff9350100c9c00 x1855008913607424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078239.729808:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913607424/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078239.729812:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078239.729814:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078239.729815:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913607424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078239.729818:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913607552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078239.729826:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078239.729831:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078239/real 1769078239] req@ffff9350100cad80 x1855008913607552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078239.729838:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913607552/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078239.729847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078239.729849:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078239.729851:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913607552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078239.921145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078240.433175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078240.433175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078240.433231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078240.433315:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078240.753219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078240.945200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078241.457148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078241.457148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078241.457194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078241.457247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078241.777148:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078241.969184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078242.481177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078242.481178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078242.481178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078242.481214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078242.801239:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078242.993169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078243.505124:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078243.505126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078243.505135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078243.505169:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078243.825156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078244.017146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078244.529178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078244.529179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078244.529179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078244.529224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078244.849163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078244.849163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078244.849173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078244.849188:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078244.849202:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3950 00000100:00080000:0.0:1769078244.849212:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078244.849219:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078244.849258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913607680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078244.849278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078244.849280:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078244.849281:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078244.849284:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3950 00000100:00080000:0.0:1769078244.849286:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078244.849288:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078244.849296:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078244.849303:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913607808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078244.849313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078244.849316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078244.849319:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078244.849323:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3950 00000100:00080000:0.0:1769078244.849327:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078244.849331:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078244.849336:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078244.849339:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913607680:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078244.849349:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913607936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078244.849361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078244.849363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078244.849365:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078244.849367:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3950 00000100:00080000:0.0:1769078244.849370:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078244.849372:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078244.849375:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078244.849385:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913608064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078244.849390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078244.849393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078244.849394:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078244.849396:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3950 00000100:00080000:0.0:1769078244.849398:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078244.849400:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078244.849402:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078244.849408:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913608192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078244.849416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078244.849418:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078244.849421:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078244.849423:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078244.849425:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3950 00000100:00080000:0.0:1769078244.849428:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078244.849430:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078244.849433:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078244.849441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913608320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078244.849446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078244.849449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078244.849450:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078244.849452:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3950 00000100:00080000:0.0:1769078244.849454:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078244.849456:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078244.849459:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078244.849467:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913608448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078244.849474:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078244/real 1769078244] req@ffff9350103faa00 x1855008913607680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078244.849494:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913607680/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078244.849504:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078244.849509:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078244.849513:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913607680:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078244.849520:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607808:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078244.849540:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078244.849550:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078244/real 1769078244] req@ffff9350103f8700 x1855008913607808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078244.849562:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913607808/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078244.849581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078244.849585:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078244.849588:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607808:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078244.849616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607936:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078244.849638:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078244.849648:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078244/real 1769078244] req@ffff9350103fb800 x1855008913607936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078244.849660:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913607936/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078244.849668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078244.849671:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078244.849674:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913607936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078244.849679:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078244.849694:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078244.849703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078244/real 1769078244] req@ffff9350103fb480 x1855008913608064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078244.849714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913608064/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078244.849721:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078244.849724:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078244.849728:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078244.849733:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913608192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078244.849750:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078244.849760:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078244/real 1769078244] req@ffff9350103fad80 x1855008913608192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078244.849771:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913608192/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078244.849779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078244.849781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078244.849785:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913608192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078244.849790:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913608320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078244.849804:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078244.849813:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078244/real 1769078244] req@ffff9350103f8380 x1855008913608320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078244.849824:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913608320/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078244.849832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078244.849835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078244.849838:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913608320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078244.849843:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913608448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078244.849857:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078244.849867:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078244/real 1769078244] req@ffff9350103fb100 x1855008913608448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078244.849879:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913608448/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078244.849887:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078244.849890:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078244.849893:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913608448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078245.041168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078245.553130:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078245.553130:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078245.553133:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078245.553179:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078245.873150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078246.065226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078246.577165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078246.577165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078246.577171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078246.577210:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078246.897195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078247.089180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078247.601217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078247.601217:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078247.601217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078247.601272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078247.921183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078248.113172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078248.625157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078248.625157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078248.625157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078248.625204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078248.945184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078249.137226:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078249.649200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078249.649200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078249.649207:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078249.649263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078249.969175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078249.969238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078249.969250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078249.969255:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078249.969265:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3955 00000100:00080000:0.0:1769078249.969275:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078249.969282:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078249.969318:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913608576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078249.969351:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078249.969355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078249.969357:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078249.969361:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3955 00000100:00080000:0.0:1769078249.969365:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078249.969368:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078249.969378:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078249.969393:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913608704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078249.969402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078249.969406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078249.969407:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078249.969414:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3955 00000100:00100000:2.0:1769078249.969414:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913608576:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078249.969420:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078249.969425:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078249.969431:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078249.969443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913608832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078249.969454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078249.969457:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078249.969459:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078249.969463:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3955 00000100:00080000:0.0:1769078249.969467:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078249.969471:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078249.969476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078249.969488:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913608960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078249.969498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078249.969502:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078249.969503:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078249.969508:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3955 00000100:00080000:0.0:1769078249.969512:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078249.969516:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608704:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078249.969519:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078249.969528:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078249.969542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913609088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078249.969552:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078249.969556:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078249.969557:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078249.969562:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3955 00000100:00080000:0.0:1769078249.969567:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078249.969567:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608832:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769078249.969576:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078249.969583:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078249.969597:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913609216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078249.969600:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608960:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769078249.969615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078249.969619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078249.969621:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078249.969625:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3955 00000100:00080000:0.0:1769078249.969630:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078249.969633:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609088:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078249.969638:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078249.969645:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078249.969658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913609344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078249.969667:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078249.969694:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078250.161174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078250.673121:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078250.673170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078250.673174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078250.673183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078250.993193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078251.185201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078251.697213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078251.697213:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078251.697213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078251.697279:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078252.017181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078252.209135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078252.721165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078252.721166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078252.721173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078252.721235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078253.041156:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:1.0:1769078253.041358:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:1.0:1769078253.041414:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769078253.041444:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078253.041493:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078253.041503:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078253.041516:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078253.041527:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078253.041541:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:1.0:1769078253.041555:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078253.041559:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078253] req@ffff9350103fbb80 x1855008913608832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078253.041593:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913608832/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078253.041610:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078253.041615:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078253.041619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078253.041627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078253] req@ffff9350103f9500 x1855008913608960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078253.041638:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913608960/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078253.041646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078253.041649:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078253.041652:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078253.041659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078253] req@ffff9350103f9f80 x1855008913609088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078253.041669:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913609088/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078253.041676:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078253.041680:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078253.041683:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609088:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078253.041690:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078253] req@ffff9350103f9180 x1855008913608576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078253.041708:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913608576/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078253.041715:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078253.041718:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078253.041721:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913608576:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078253.041733:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078253] req@ffff9350103f9c00 x1855008913608704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078253.041744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913608704/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078253.041753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078253.041756:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078253.041760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913608704:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078253.041799:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078253] req@ffff9350103f8380 x1855008913609344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078253.041811:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913609344/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078253.041828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078253.041831:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078253.041834:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078253.041841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078253] req@ffff9350103fb100 x1855008913609216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078253.041855:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913609216/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078253.041862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078253.041865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078253.041869:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078253.233181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078253.745209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078253.745209:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078253.745213:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078253.745271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078254.065176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078254.257195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078254.769222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078254.769223:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078254.769235:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078254.769271:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078255.089165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078255.089166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078255.089176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078255.089180:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078255.089191:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3960 00000100:00080000:0.0:1769078255.089202:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078255.089214:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078255.089247:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913609472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078255.089275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078255.089279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078255.089280:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078255.089284:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3960 00000100:00080000:0.0:1769078255.089288:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078255.089293:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078255.089309:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078255.089326:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913609600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078255.089337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078255.089340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078255.089342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078255.089346:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3960 00000100:00080000:0.0:1769078255.089349:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078255.089354:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078255.089360:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078255.089377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913609728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078255.089381:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913609472:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078255.089396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078255.089403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078255.089406:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078255.089411:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3960 00000100:00080000:0.0:1769078255.089415:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078255.089421:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078255.089426:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078255.089440:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913609856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078255.089450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078255.089454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078255.089455:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078255.089460:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3960 00000100:00080000:0.0:1769078255.089465:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078255.089471:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078255.089476:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078255.089486:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078255.089492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913609984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078255.089506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078255.089510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078255.089512:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078255.089519:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3960 00000100:00080000:0.0:1769078255.089524:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078255.089528:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078255.089535:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078255.089554:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913610112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078255.089559:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078255/real 1769078255] req@ffff9350103fad80 x1855008913609472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078255.089574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078255.089593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078255.089599:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078255.089608:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3960 00000100:00080000:0.0:1769078255.089613:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078255.089619:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913609472/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078255.089626:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078255.089641:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078255.089648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769078255.089665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913610240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078255.089667:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078255.089686:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913609472:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078255.089700:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913609600:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078255.089754:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078255.089771:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078255/real 1769078255] req@ffff9350103fb480 x1855008913609600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078255.089783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913609600/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078255.089792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078255.089795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078255.089798:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913609600:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078255.089826:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913609728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078255.089843:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078255.089855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078255/real 1769078255] req@ffff9350103fb800 x1855008913609728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078255.089865:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913609728/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078255.089873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078255.089876:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078255.089879:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913609728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078255.089885:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913609856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078255.089910:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078255.089920:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078255/real 1769078255] req@ffff9350103f8700 x1855008913609856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078255.089930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913609856/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078255.089937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078255.089940:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078255.089943:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913609856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078255.089950:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609984:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078255.089969:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078255.089978:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078255/real 1769078255] req@ffff9350103faa00 x1855008913609984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078255.089989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913609984/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078255.089997:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078255.090000:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078255.090003:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913609984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078255.090040:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913610112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078255.090112:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078255.090122:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078255/real 1769078255] req@ffff9350103fb100 x1855008913610112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078255.090132:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913610112/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078255.090140:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078255.090144:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078255.090147:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913610112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078255.090153:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913610240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078255.090169:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078255.090180:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078255/real 1769078255] req@ffff9350103f8380 x1855008913610240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078255.090189:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913610240/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078255.090202:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078255.090205:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078255.090208:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913610240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078255.281208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078255.793182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078255.793183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078255.793185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078255.793220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078256.113149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078256.305173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078256.817190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078256.817192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078256.817192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078256.817335:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078257.137165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078257.329134:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078257.841212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078257.841216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078257.841224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078257.841264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078258.161142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078258.353153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078258.865160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078258.865160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078258.865168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078258.865234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078259.185212:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078259.377170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078259.889182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078259.889184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078259.889235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078259.889250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078260.209202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078260.209204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078260.209223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078260.209227:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078260.209235:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3965 00000100:00080000:3.0:1769078260.209245:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078260.209251:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078260.209283:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913610368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078260.209304:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078260.209307:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078260.209309:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078260.209313:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3965 00000100:00080000:3.0:1769078260.209317:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078260.209321:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078260.209330:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078260.209344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913610496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078260.209355:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913610368:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078260.209357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078260.209365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078260.209367:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078260.209370:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3965 00000100:00080000:3.0:1769078260.209375:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078260.209378:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078260.209382:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078260.209393:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913610624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078260.209400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078260.209402:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078260.209403:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078260.209405:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3965 00000100:00080000:3.0:1769078260.209415:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078260.209417:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078260.209421:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078260.209422:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078260.209431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913610752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078260.209439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078260.209441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078260.209443:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078260.209445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3965 00000100:00080000:3.0:1769078260.209447:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078260.209449:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078260.209452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078260.209462:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913610880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078260.209468:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078260/real 1769078260] req@ffff935002a7e300 x1855008913610368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078260.209469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078260.209476:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078260.209479:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078260.209486:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3965 00000100:00080000:3.0:1769078260.209491:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078260.209493:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078260.209496:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078260.209501:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913610368/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078260.209504:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913611008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078260.209528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078260.209530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078260.209531:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078260.209532:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078260.209539:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3965 00000100:00080000:2.0:1769078260.209542:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078260.209543:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078260.209548:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913610368:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078260.209556:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078260.209559:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078260.209566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913611136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078260.209573:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913610496:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078260.209606:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078260.209618:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078260/real 1769078260] req@ffff935002a7d180 x1855008913610496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078260.209629:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913610496/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078260.209636:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078260.209639:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078260.209643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913610496:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078260.209648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913610624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078260.209664:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078260.209674:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078260/real 1769078260] req@ffff935002a7dc00 x1855008913610624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078260.209684:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913610624/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078260.209691:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078260.209694:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078260.209697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913610624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078260.209710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913610752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078260.209726:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078260.209734:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078260/real 1769078260] req@ffff935002a7c380 x1855008913610752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078260.209745:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913610752/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078260.209752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078260.209754:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078260.209757:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913610752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078260.209762:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913610880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078260.209776:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078260.209785:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078260/real 1769078260] req@ffff935002a7fb80 x1855008913610880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078260.209795:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913610880/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078260.209803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078260.209805:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078260.209808:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913610880:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078260.209813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078260.209828:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078260.209837:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078260/real 1769078260] req@ffff935002a7c700 x1855008913611008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078260.209847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913611008/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078260.209854:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078260.209857:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078260.209860:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078260.209878:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078260.209897:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078260.209910:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078260/real 1769078260] req@ffff935002a7f100 x1855008913611136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078260.209939:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913611136/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078260.209948:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078260.209950:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078260.209954:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078260.401099:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078260.913146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078260.913201:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078260.913208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078260.913262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078261.233154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078261.425155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078261.937165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078261.937188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078261.937207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078261.937216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078262.257210:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078262.449188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078262.961125:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078262.961125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078262.961149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078262.961246:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078263.281203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078263.473212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078263.985197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078263.985198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078263.985198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078263.985218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078264.305173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078264.497189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078265.009200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078265.009203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078265.009224:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078265.009229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078265.329134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078265.329139:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078265.329144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078265.329150:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078265.329160:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3971 00000100:00080000:3.0:1769078265.329171:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078265.329178:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078265.329215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913611264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078265.329242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078265.329245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078265.329246:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078265.329249:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3971 00000100:00080000:3.0:1769078265.329251:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078265.329253:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078265.329262:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078265.329270:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913611392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078265.329280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078265.329283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078265.329285:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078265.329288:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3971 00000100:00080000:3.0:1769078265.329290:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078265.329292:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078265.329295:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078265.329301:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913611520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078265.329311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078265.329313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078265.329315:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078265.329321:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3971 00000100:00080000:3.0:1769078265.329323:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078265.329324:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913611264:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078265.329326:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078265.329332:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078265.329342:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913611648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078265.329348:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078265.329350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078265.329351:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078265.329354:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3971 00000100:00080000:3.0:1769078265.329356:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078265.329359:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078265.329362:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078265.329371:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913611776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078265.329376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078265.329378:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078265.329380:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078265.329383:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3971 00000100:00080000:3.0:1769078265.329394:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078265.329397:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078265.329401:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078265.329405:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078265.329410:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913611904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078265.329421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078265.329424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078265.329425:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078265.329428:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3971 00000100:00080000:3.0:1769078265.329432:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078265.329435:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078265.329440:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078265.329448:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078265/real 1769078265] req@ffff935002a7df80 x1855008913611264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078265.329449:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913612032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078265.329514:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913611264/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078265.329535:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078265.329540:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078265.329544:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913611264:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078265.329550:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913611392:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078265.329568:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078265.329577:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078265/real 1769078265] req@ffff935002a7f800 x1855008913611392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078265.329585:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913611392/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078265.329594:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078265.329596:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078265.329598:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913611392:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078265.329618:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913611520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078265.329628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078265.329641:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078265/real 1769078265] req@ffff935002a7c000 x1855008913611520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078265.329648:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913611520/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078265.329652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078265.329654:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078265.329656:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913611520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078265.329659:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913611648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078265.329668:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078265.329673:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078265/real 1769078265] req@ffff935002a7ca80 x1855008913611648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078265.329680:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913611648/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078265.329684:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078265.329685:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078265.329688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913611648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078265.329691:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611776:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078265.329700:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078265.329704:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078265/real 1769078265] req@ffff935002a7f100 x1855008913611776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078265.329710:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913611776/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078265.329717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078265.329720:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078265.329723:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078265.329729:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611904:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078265.329746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078265.329756:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078265/real 1769078265] req@ffff935002a7c700 x1855008913611904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078265.329767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913611904/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078265.329773:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078265.329776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078265.329777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913611904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078265.329781:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078265.329794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078265.329800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078265/real 1769078265] req@ffff935002a7fb80 x1855008913612032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078265.329806:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913612032/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078265.329811:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078265.329813:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078265.329816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078265.521167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078266.033184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078266.033188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078266.033188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078266.033214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078266.353178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078266.545205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078267.057185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078267.057185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078267.057185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078267.057216:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078267.377154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078267.569176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078268.081154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078268.081154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078268.081164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078268.081182:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078268.401140:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078268.593195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078269.105188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078269.105197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078269.105203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078269.105235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078269.425203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078269.617198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078270.129143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078270.129170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078270.129184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078270.129316:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078270.449155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078270.449155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078270.449166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078270.449170:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078270.449179:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3976 00000100:00080000:3.0:1769078270.449190:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078270.449197:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078270.449234:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913612160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078270.449263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078270.449267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078270.449269:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078270.449273:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3976 00000100:00080000:3.0:1769078270.449277:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078270.449281:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078270.449290:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078270.449303:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913612288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078270.449314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078270.449318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078270.449320:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078270.449325:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3976 00000100:00080000:3.0:1769078270.449329:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078270.449333:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078270.449339:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078270.449351:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913612160:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078270.449354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913612416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078270.449377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078270.449381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078270.449383:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078270.449389:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3976 00000100:00080000:3.0:1769078270.449394:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078270.449398:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078270.449404:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078270.449417:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913612544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078270.449435:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769078270.449446:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078270.449446:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078270.449452:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078270.449460:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3976 00000100:00080000:3.0:1769078270.449466:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078270.449471:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078270.449477:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078270.449494:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913612672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078270.449503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078270.449508:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078270/real 1769078270] req@ffff935002a7c380 x1855008913612160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078270.449527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078270.449533:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078270.449540:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3976 00000100:00080000:3.0:1769078270.449545:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078270.449549:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913612160/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078270.449557:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078270.449572:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078270.449576:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078270.449588:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078270.449588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913612800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078270.449605:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913612160:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078270.449607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078270.449615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078270.449621:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078270.449626:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3976 00000100:00100000:2.0:1769078270.449631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913612288:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078270.449633:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078270.449645:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078270.449651:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078270.449667:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913612928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078270.449673:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078270.449699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078270/real 1769078270] req@ffff935002a7dc00 x1855008913612288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078270.449714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913612288/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078270.449723:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078270.449727:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078270.449730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913612288:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078270.449777:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913612416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078270.449799:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078270.449811:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078270/real 1769078270] req@ffff935002a7d180 x1855008913612416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078270.449824:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913612416/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078270.449832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078270.449835:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078270.449839:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913612416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078270.449845:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913612544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078270.449861:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078270.449875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078270/real 1769078270] req@ffff935002a7e300 x1855008913612544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078270.449887:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913612544/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078270.449896:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078270.449900:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078270.449904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913612544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078270.449909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612672:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078270.449927:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078270.449936:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078270/real 1769078270] req@ffff935002a7fb80 x1855008913612672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078270.449947:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913612672/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078270.449955:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078270.449958:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078270.449962:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078270.449968:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078270.449986:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078270.449997:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078270/real 1769078270] req@ffff935002a7c700 x1855008913612800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078270.450008:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913612800/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078270.450016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078270.450019:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078270.450023:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078270.450030:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078270.450046:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078270.450104:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078270/real 1769078270] req@ffff935002a7f100 x1855008913612928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078270.450116:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913612928/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078270.450123:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078270.450126:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078270.450129:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913612928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078270.641175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078271.153166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078271.153170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078271.153221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078271.153302:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078271.473179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078271.665146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078272.177164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078272.177165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078272.177255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078272.177308:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078272.497178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078272.689157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078273.201160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078273.201182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078273.201191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078273.201201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078273.521161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078273.713107:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078274.225177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078274.225178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078274.225182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078274.225318:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078274.545168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078274.737166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078275.249184:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078275.249758:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078275.249796:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078275.249900:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078275.569163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078275.569177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078275.569184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078275.569189:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078275.569197:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3981 00000100:00080000:2.0:1769078275.569208:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078275.569213:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078275.569247:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913613056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078275.569276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078275.569281:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078275.569282:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078275.569286:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3981 00000100:00080000:2.0:1769078275.569300:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078275.569305:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078275.569315:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078275.569316:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913613056:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078275.569331:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913613184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078275.569342:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078275.569346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078275.569348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078275.569352:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3981 00000100:00080000:2.0:1769078275.569355:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078275.569359:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078275.569364:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078275.569375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913613312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078275.569385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769078275.569387:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078275.569390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078275.569394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078275.569398:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3981 00000100:00080000:2.0:1769078275.569403:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078275.569406:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078275.569412:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078275.569424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913613440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078275.569432:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078275/real 1769078275] req@ffff935002a63100 x1855008913613056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078275.569437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078275.569453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1769078275.569455:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913613056/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078275.569456:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078275.569464:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3981 00000100:00080000:2.0:1769078275.569471:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078275.569473:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078275.569478:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769078275.569482:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769078275.569485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078275.569489:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913613056:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078275.569500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913613568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078275.569516:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913613184:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769078275.569519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078275.569524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078275.569526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078275.569535:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3981 00000100:00080000:2.0:1769078275.569540:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078275.569545:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078275.569547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078275.569554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078275.569564:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078275/real 1769078275] req@ffff935002a61500 x1855008913613184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078275.569568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913613696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078275.569591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078275.569594:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913613184/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078275.569596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078275.569599:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078275.569608:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3981 00000100:00080000:2.0:1769078275.569614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078275.569616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078275.569618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769078275.569625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000008:00100000:2.0:1769078275.569626:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078275.569633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913613184:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078275.569640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913613824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078275.569645:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913613312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078275.569693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078275.569705:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078275/real 1769078275] req@ffff935002a60000 x1855008913613312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078275.569713:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913613312/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078275.569723:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078275.569725:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078275.569727:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913613312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078275.569732:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913613440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078275.569747:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078275.569753:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078275/real 1769078275] req@ffff935002a62a00 x1855008913613440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078275.569761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913613440/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078275.569766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078275.569768:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078275.569769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913613440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078275.569780:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913613568:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078275.569789:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078275.569794:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078275/real 1769078275] req@ffff935002a61c00 x1855008913613568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078275.569802:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913613568/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078275.569807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078275.569809:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078275.569810:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913613568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078275.569813:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913613696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078275.569821:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078275.569826:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078275/real 1769078275] req@ffff935002a61f80 x1855008913613696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078275.569835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913613696/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078275.569843:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078275.569846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078275.569849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913613696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078275.569855:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913613824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078275.569872:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078275.569882:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078275/real 1769078275] req@ffff935002a60a80 x1855008913613824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078275.569893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913613824/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078275.569906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078275.569909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078275.569912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913613824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078275.761181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078276.273185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078276.273186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078276.273186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078276.273209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078276.593172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078276.785150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078277.297227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078277.297227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078277.297231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078277.297274:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078277.617157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078277.809145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078278.321154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078278.321154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078278.321154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078278.321276:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078278.641170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078278.833164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078279.345154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078279.345190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078279.345234:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078279.345293:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078279.665091:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078279.857215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078280.369176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078280.369182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078280.369206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078280.369243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078280.689165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078280.689187:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078280.689209:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078280.689216:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078280.689226:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3986 00000100:00080000:3.0:1769078280.689237:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078280.689243:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078280.689288:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913613952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078280.689322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078280.689326:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078280.689328:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078280.689332:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3986 00000100:00080000:3.0:1769078280.689336:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078280.689340:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078280.689362:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078280.689378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913614080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078280.689391:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913613952:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078280.689394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078280.689405:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078280.689407:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078280.689412:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3986 00000100:00080000:3.0:1769078280.689416:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078280.689421:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078280.689427:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078280.689440:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913614208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078280.689450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078280.689453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078280.689455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078280.689459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3986 00000100:00080000:3.0:1769078280.689464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078280.689468:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078280.689473:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078280.689483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913614336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078280.689493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078280.689496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078280.689498:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078280.689502:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3986 00000100:00080000:3.0:1769078280.689506:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078280.689510:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078280.689524:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078280.689536:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078280.689539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913614464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078280.689549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078280.689553:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078280.689555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078280.689559:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3986 00000100:00080000:3.0:1769078280.689563:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078280.689567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078280.689584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078280.689596:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078280/real 1769078280] req@ffff935002a7ca80 x1855008913613952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078280.689604:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913614592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078280.689614:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913613952/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078280.689633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078280.689635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078280.689641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078280.689643:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078280.689645:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078280.689647:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913613952:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078280.689652:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3986 00000100:00080000:3.0:1769078280.689657:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078280.689662:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078280.689667:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078280.689672:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078280.689680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913614720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078280.689707:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078280.689722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078280/real 1769078280] req@ffff935002a7c000 x1855008913614080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078280.689742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913614080/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078280.689748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078280.689751:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078280.689753:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614080:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078280.689757:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078280.689772:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078280.689779:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078280/real 1769078280] req@ffff935002a7f800 x1855008913614208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078280.689787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913614208/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078280.689791:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078280.689793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078280.689796:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078280.689799:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078280.689807:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078280.689812:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078280/real 1769078280] req@ffff935002a7df80 x1855008913614336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078280.689819:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913614336/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078280.689828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078280.689830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078280.689831:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078280.689834:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913614464:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078280.689842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078280.689847:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078280/real 1769078280] req@ffff935002a7f100 x1855008913614464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078280.689853:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913614464/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078280.689858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078280.689860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078280.689862:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913614464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078280.689864:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913614592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078280.689873:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078280.689877:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078280/real 1769078280] req@ffff935002a7c700 x1855008913614592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078280.689884:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913614592/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078280.689891:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078280.689893:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078280.689897:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913614592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078280.689917:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913614720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078280.689936:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078280.689947:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078280/real 1769078280] req@ffff935002a7fb80 x1855008913614720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078280.689955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913614720/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078280.689960:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078280.689961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078280.689963:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913614720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078280.881177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078281.393165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078281.393165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078281.393165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078281.393194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078281.713180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078281.905169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078282.417131:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078282.417138:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078282.417142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078282.417199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078282.737187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078282.929181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078283.441170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078283.441190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078283.441190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078283.441224:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078283.761200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078283.953202:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078284.465176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078284.465177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078284.465177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078284.465239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078284.785146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078284.977153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078285.489164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078285.489164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078285.489169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078285.489202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078285.809175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078285.809209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078285.809214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078285.809217:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078285.809226:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3991 00000100:00080000:1.0:1769078285.809236:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078285.809243:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078285.809282:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913614848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078285.809308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078285.809311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078285.809313:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078285.809317:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3991 00000100:00080000:1.0:1769078285.809321:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078285.809326:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078285.809338:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078285.809359:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913614848:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078285.809360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913614976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078285.809372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078285.809374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078285.809376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078285.809379:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3991 00000100:00080000:1.0:1769078285.809382:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078285.809385:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078285.809389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078285.809399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913615104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078285.809405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078285.809407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078285.809408:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078285.809410:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3991 00000100:00080000:1.0:1769078285.809414:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078285.809416:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078285.809419:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078285.809425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913615232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078285.809430:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078285.809442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078285.809446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078285.809448:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078285.809453:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3991 00000100:00080000:1.0:1769078285.809457:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078285.809461:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078285.809467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078285.809473:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078285/real 1769078285] req@ffff9350100c9880 x1855008913614848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078285.809480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913615360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078285.809505:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913614848/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078285.809516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078285.809517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078285.809522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078285.809524:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078285.809524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078285.809531:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913614848:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078285.809532:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3991 00000100:00080000:1.0:1769078285.809537:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078285.809541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078285.809545:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078285.809553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913615488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078285.809558:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614976:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078285.809565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078285.809568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078285.809569:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078285.809572:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3991 00000100:00080000:1.0:1769078285.809574:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078285.809577:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078285.809580:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078285.809584:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769078285.809589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913615616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078285.809597:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078285/real 1769078285] req@ffff9350100c9500 x1855008913614976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078285.809607:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913614976/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078285.809613:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078285.809616:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078285.809619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913614976:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078285.809624:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913615104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078285.809635:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078285.809640:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078285/real 1769078285] req@ffff9350100caa00 x1855008913615104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078285.809646:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913615104/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078285.809651:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078285.809652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078285.809654:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913615104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078285.809657:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913615232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078285.809665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078285.809670:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078285/real 1769078285] req@ffff9350100c8380 x1855008913615232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078285.809677:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913615232/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078285.809681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078285.809683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078285.809684:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913615232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078285.809687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913615360:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078285.809695:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078285.809700:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078285/real 1769078285] req@ffff9350100ca680 x1855008913615360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078285.809705:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913615360/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078285.809710:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078285.809711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078285.809713:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913615360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078285.809726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913615488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078285.809742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078285.809753:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078285/real 1769078285] req@ffff9350100c8e00 x1855008913615488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078285.809770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913615488/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078285.809778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078285.809781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078285.809784:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913615488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078285.809789:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913615616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078285.809806:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078285.809817:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078285/real 1769078285] req@ffff9350100ca300 x1855008913615616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078285.809828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913615616/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078285.809835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078285.809838:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078285.809841:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913615616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078286.001170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078286.513186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078286.513310:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078286.513390:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078286.513405:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078286.833185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078287.025153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078287.537172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078287.537176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078287.537209:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078287.537239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078287.857211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078288.049166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078288.561168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078288.561168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078288.561168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078288.561202:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078288.881185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078289.073173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078289.585139:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078289.585141:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078289.585141:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078289.585174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078289.905188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078290.097173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078290.609186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078290.609186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078290.609220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078290.609248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078290.929213:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078290.929226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078290.929227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078290.929234:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078290.929246:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 3996 00000100:00080000:0.0:1769078290.929259:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078290.929268:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078290.929325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913615744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078290.929359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078290.929363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078290.929364:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078290.929369:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3996 00000100:00080000:0.0:1769078290.929372:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078290.929376:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078290.929389:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078290.929420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913615872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078290.929429:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913615744:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078290.929435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078290.929440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078290.929442:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078290.929447:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3996 00000100:00080000:0.0:1769078290.929451:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078290.929455:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078290.929461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078290.929473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913616000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078290.929488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078290.929492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078290.929494:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078290.929498:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 3996 00000100:00080000:0.0:1769078290.929502:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078290.929505:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078290.929510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078290.929514:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078290.929534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913616128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078290.929543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078290.929547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078290.929549:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078290.929554:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3996 00000100:00080000:0.0:1769078290.929558:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078290.929563:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078290.929568:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078290.929568:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9350103f9c00 x1855008913615744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078290.929583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913616256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078290.929593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078290.929595:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913615744/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078290.929598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078290.929601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078290.929611:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3996 00000100:00080000:2.0:1769078290.929616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078290.929617:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078290.929624:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078290.929631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078290.929637:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078290.929644:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913615744:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078290.929645:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913616384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078290.929667:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078290.929670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078290.929672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078290.929675:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913615872:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078290.929680:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 3996 00000100:00080000:0.0:1769078290.929685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078290.929689:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078290.929695:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078290.929707:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078290.929709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913616512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078290.929725:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9350103f9180 x1855008913615872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078290.929742:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913615872/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078290.929753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078290.929756:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078290.929760:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913615872:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078290.929767:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078290.929795:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078290.929808:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9350103f9f80 x1855008913616000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078290.929820:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913616000/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078290.929827:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078290.929830:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078290.929833:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078290.929840:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616128:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078290.929857:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078290.929879:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9350103f9500 x1855008913616128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078290.929891:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913616128/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078290.929899:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078290.929902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078290.929905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078290.929911:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913616256:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078290.929930:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078290.929940:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9350103fbb80 x1855008913616256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078290.929952:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913616256/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078290.929959:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078290.929962:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078290.929965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913616256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078290.929970:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913616384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078290.929998:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078290.930008:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9350103f8380 x1855008913616384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078290.930026:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913616384/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078290.930034:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078290.930036:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078290.930040:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913616384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078290.930087:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913616512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078290.930105:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078290.930115:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9350103fb100 x1855008913616512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078290.930125:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913616512/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078290.930133:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078290.930135:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078290.930138:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913616512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078291.121105:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078291.633154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078291.633154:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078291.633154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078291.633200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078291.953169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078292.145190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078292.657182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078292.657182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078292.657182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078292.657218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078292.977199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078293.169187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078293.681193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078293.681193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078293.681193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078293.681381:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078294.001233:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078294.193190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078294.705180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078294.705180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078294.705295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078294.705302:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078295.025108:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078295.217203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078295.729136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078295.729148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078295.729190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078295.729192:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078296.049167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078296.049180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1769078296.049183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078296.049185:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078296.049197:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4001 00000100:00080000:0.0:1769078296.049207:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078296.049214:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078296.049244:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913616640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078296.049272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078296.049276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078296.049277:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078296.049282:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4001 00000100:00080000:0.0:1769078296.049286:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078296.049291:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078296.049300:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078296.049323:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913616768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078296.049335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078296.049335:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913616640:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078296.049354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078296.049357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078296.049360:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4001 00000100:00080000:0.0:1769078296.049363:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078296.049367:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078296.049374:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078296.049386:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913616896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078296.049392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078296.049394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078296.049395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078296.049400:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4001 00000100:00080000:0.0:1769078296.049402:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078296.049404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078296.049408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078296.049421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913617024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078296.049427:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078296.049443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078296.049448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078296.049450:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078296.049458:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4001 00000100:00080000:0.0:1769078296.049463:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078296.049467:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078296.049473:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078296.049478:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078296/real 1769078296] req@ffff9350103faa00 x1855008913616640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078296.049487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913617152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078296.049521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078296.049524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078296.049532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078296.049532:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913616640/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078296.049546:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4001 00000100:00080000:0.0:1769078296.049558:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078296.049570:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078296.049571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769078296.049576:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078296.049578:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078296.049586:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913616640:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078296.049593:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913617280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078296.049601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078296.049604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078296.049605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078296.049607:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4001 00000100:00080000:0.0:1769078296.049609:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078296.049612:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078296.049615:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078296.049619:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078296.049627:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913617408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078296.049655:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078296.049672:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078296/real 1769078296] req@ffff9350103f8700 x1855008913616768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078296.049683:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913616768/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078296.049691:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078296.049695:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078296.049697:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616768:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078296.049704:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078296.049720:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078296.049728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078296/real 1769078296] req@ffff9350103fb800 x1855008913616896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078296.049738:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913616896/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078296.049745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078296.049748:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078296.049751:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913616896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078296.049756:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078296.049770:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078296.049778:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078296/real 1769078296] req@ffff9350103fb480 x1855008913617024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078296.049788:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913617024/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078296.049794:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078296.049798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078296.049801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078296.049806:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913617152:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078296.049823:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078296.049832:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078296/real 1769078296] req@ffff9350103fad80 x1855008913617152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078296.049842:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913617152/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078296.049850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078296.049853:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078296.049856:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913617152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078296.049862:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913617280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078296.049877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078296.049887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078296/real 1769078296] req@ffff9350103fb100 x1855008913617280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078296.049898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913617280/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078296.049903:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078296.049905:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078296.049907:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913617280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078296.049921:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913617408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078296.049933:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078296.049939:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078296/real 1769078296] req@ffff9350103f8380 x1855008913617408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078296.049945:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913617408/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078296.049950:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078296.049952:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078296.049954:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913617408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078296.241123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078296.753189:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078296.753189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078296.753198:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078296.753283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078297.073121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078297.265189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078297.777176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078297.777248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078297.777331:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078297.777400:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078298.097195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078298.289131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078298.801149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078298.801149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078298.801173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078298.801376:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078299.121162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078299.313162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078299.825120:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078299.825168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078299.825181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078299.825212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078300.145142:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078300.337171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078300.849169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078300.849169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078300.849169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078300.849220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078301.169153:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078301.169174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078301.169180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078301.169183:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078301.169190:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4006 00000100:00080000:0.0:1769078301.169199:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078301.169205:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078301.169267:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913617536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078301.169285:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078301.169288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078301.169290:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078301.169297:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4006 00000100:00080000:0.0:1769078301.169300:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078301.169304:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078301.169314:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078301.169322:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913617536:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078301.169335:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913617664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078301.169347:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078301.169351:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078301.169352:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078301.169355:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4006 00000100:00080000:0.0:1769078301.169358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078301.169361:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078301.169366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078301.169374:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913617792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078301.169381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078301.169383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078301.169384:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078301.169387:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4006 00000100:00080000:0.0:1769078301.169390:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078301.169392:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078301.169394:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078301.169400:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913617920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078301.169406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078301.169408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078301.169409:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078301.169412:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4006 00000100:00080000:0.0:1769078301.169414:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078301.169421:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078301.169422:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078301.169430:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078301.169437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913618048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078301.169443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078301.169446:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078301.169448:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078301.169450:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4006 00000100:00080000:0.0:1769078301.169452:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078301.169462:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078301.169466:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078301.169480:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913618176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078301.169484:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078301/real 1769078301] req@ffff9350103fbb80 x1855008913617536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078301.169507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078301.169530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078301.169533:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078301.169538:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4006 00000100:00080000:0.0:1769078301.169541:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078301.169545:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913617536/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078301.169550:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078301.169561:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078301.169578:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769078301.169579:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913618304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078301.169594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078301.169604:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913617536:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078301.169640:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617664:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078301.169665:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078301.169677:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078301/real 1769078301] req@ffff9350103f9500 x1855008913617664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078301.169689:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913617664/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078301.169699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078301.169703:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078301.169706:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617664:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078301.169712:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078301.169730:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078301.169741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078301/real 1769078301] req@ffff9350103f9f80 x1855008913617792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078301.169752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913617792/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078301.169765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078301.169768:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078301.169771:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078301.169777:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078301.169794:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078301.169804:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078301/real 1769078301] req@ffff9350103f9180 x1855008913617920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078301.169815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913617920/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078301.169823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078301.169826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078301.169829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913617920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078301.169835:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618048:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078301.169853:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078301.169863:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078301/real 1769078301] req@ffff9350103f9c00 x1855008913618048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078301.169875:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913618048/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078301.169883:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078301.169886:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078301.169890:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618048:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078301.169896:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618176:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078301.169911:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078301.169920:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078301/real 1769078301] req@ffff9350103f8380 x1855008913618176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078301.169932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913618176/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078301.169940:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078301.169943:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078301.169946:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078301.169952:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078301.169968:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078301.169977:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078301/real 1769078301] req@ffff9350103fb100 x1855008913618304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078301.169988:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913618304/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078301.169995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078301.169999:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078301.170002:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078301.361146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078301.873116:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078301.873118:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078301.873158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078301.873230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078302.193149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078302.385177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078302.897155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078302.897165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078302.897218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078302.897232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078303.217168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078303.409098:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078303.921189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078303.921192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078303.921220:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078303.921245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078304.241121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078304.433209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078304.945160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078304.945160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078304.945160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078304.945204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078305.265188:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078305.457191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078305.969184:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078305.969184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078305.969217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078305.969314:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078306.289163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078306.289169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078306.289179:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078306.289184:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078306.289196:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4011 00000100:00080000:0.0:1769078306.289207:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078306.289215:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078306.289249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913618432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078306.289285:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078306.289288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078306.289290:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078306.289294:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4011 00000100:00080000:0.0:1769078306.289297:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078306.289301:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078306.289311:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078306.289327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913618560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078306.289339:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078306.289347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769078306.289348:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913618432:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078306.289352:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078306.289364:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4011 00000100:00080000:0.0:1769078306.289367:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078306.289372:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078306.289379:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078306.289392:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913618688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078306.289400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078306.289404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078306.289406:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078306.289409:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4011 00000100:00080000:0.0:1769078306.289413:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078306.289419:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078306.289423:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1769078306.289429:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078306.289437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913618816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078306.289449:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078306.289452:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078306.289454:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078306.289459:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4011 00000100:00080000:0.0:1769078306.289463:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078306.289468:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078306.289473:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078306.289481:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078306/real 1769078306] req@ffff9350103fad80 x1855008913618432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078306.289488:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913618944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078306.289506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769078306.289507:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913618432/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078306.289516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078306.289522:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078306.289530:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078306.289532:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4011 00000100:00080000:0.0:1769078306.289538:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078306.289538:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078306.289546:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913618432:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078306.289548:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078306.289562:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078306.289576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913619072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078306.289589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769078306.289589:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913618560:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078306.289604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078306.289608:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078306.289615:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4011 00000100:00080000:0.0:1769078306.289620:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769078306.289628:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078306.289629:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078306.289637:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078306.289649:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078306/real 1769078306] req@ffff9350103fb480 x1855008913618560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078306.289652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913619200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078306.289676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913618560/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078306.289683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078306.289686:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078306.289691:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913618560:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078306.289697:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913618688:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078306.289713:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078306.289720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078306/real 1769078306] req@ffff9350103fb800 x1855008913618688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078306.289733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913618688/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078306.289738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078306.289740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078306.289742:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913618688:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078306.289745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913618816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078306.289755:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078306.289761:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078306/real 1769078306] req@ffff9350103f8700 x1855008913618816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078306.289767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913618816/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078306.289772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078306.289773:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078306.289775:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913618816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078306.289778:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618944:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769078306.289792:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078306.289798:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078306/real 1769078306] req@ffff9350103faa00 x1855008913618944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078306.289805:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913618944/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078306.289810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078306.289811:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078306.289813:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913618944:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078306.289825:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078306.289835:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078306.289840:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078306/real 1769078306] req@ffff9350103fb100 x1855008913619072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078306.289847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913619072/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078306.289851:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078306.289853:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078306.289854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078306.289857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078306.289865:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078306.289875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078306/real 1769078306] req@ffff9350103f8380 x1855008913619200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078306.289881:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913619200/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078306.289889:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078306.289891:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078306.289893:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078306.481208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078306.993153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078306.993173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078306.993208:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078306.993239:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078307.313190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078307.505191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078308.017179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078308.017179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078308.017236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078308.017302:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078308.337169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078308.529146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078309.041126:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078309.041142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078309.041156:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078309.041248:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078309.361161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078309.553192:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078310.065185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078310.065185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078310.065185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078310.065247:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078310.385193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078310.577136:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078311.089203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078311.089203:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078311.089203:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078311.089270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078311.409161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078311.409171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078311.409175:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078311.409179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078311.409185:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4017 00000100:00080000:1.0:1769078311.409197:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078311.409205:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078311.409244:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913619328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078311.409266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078311.409270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078311.409272:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078311.409275:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4017 00000100:00080000:1.0:1769078311.409277:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078311.409279:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078311.409288:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078311.409299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913619456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078311.409306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078311.409309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078311.409310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078311.409315:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4017 00000100:00100000:3.0:1769078311.409316:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913619328:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078311.409317:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078311.409323:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078311.409328:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078311.409333:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913619584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078311.409339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078311.409342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078311.409343:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078311.409345:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4017 00000100:00080000:1.0:1769078311.409347:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078311.409349:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078311.409359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078311.409368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913619712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078311.409378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078311.409382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078311.409383:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078311.409387:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4017 00000100:00080000:1.0:1769078311.409391:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769078311.409396:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078311.409399:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078311.409406:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078311.409418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913619840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078311.409426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078311.409428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078311.409430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078311.409433:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4017 00000100:00080000:1.0:1769078311.409436:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078311.409438:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078311.409447:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078311/real 1769078311] req@ffff9350100cb800 x1855008913619328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769078311.409448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078311.409461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913619968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078311.409468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078311.409470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078311.409474:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769078311.409475:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913619328/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078311.409478:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4017 00000100:00080000:1.0:1769078311.409482:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078311.409487:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078311.409492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769078311.409497:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769078311.409502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913620096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078311.409507:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078311.409514:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913619328:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078311.409540:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913619456:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769078311.409571:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078311.409584:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078311/real 1769078311] req@ffff9350100ca300 x1855008913619456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078311.409601:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913619456/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078311.409610:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078311.409613:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078311.409615:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913619456:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078311.409621:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913619584:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078311.409639:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078311.409649:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078311/real 1769078311] req@ffff9350100c8e00 x1855008913619584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078311.409660:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913619584/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078311.409667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078311.409670:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078311.409673:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913619584:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078311.409678:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913619712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078311.409705:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078311.409715:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078311/real 1769078311] req@ffff9350100ca680 x1855008913619712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078311.409726:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913619712/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078311.409737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078311.409740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078311.409743:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913619712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078311.409748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619840:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769078311.409764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078311.409773:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078311/real 1769078311] req@ffff9350100c8380 x1855008913619840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078311.409783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913619840/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078311.409790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078311.409793:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078311.409796:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619840:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078311.409801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078311.409817:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078311.409826:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078311/real 1769078311] req@ffff9350100caa00 x1855008913619968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078311.409846:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913619968/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078311.409854:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078311.409857:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078311.409860:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913619968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078311.409867:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078311.409887:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078311.409897:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078311/real 1769078311] req@ffff9350100c9500 x1855008913620096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078311.409908:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913620096/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078311.409915:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078311.409917:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078311.409920:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078311.601204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078312.113204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078312.113204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078312.113204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078312.113245:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078312.433302:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078312.625115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078313.137127:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078313.137132:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078313.137140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078313.137167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078313.457121:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078313.649135:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078314.161161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078314.161161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078314.161161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078314.161191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078314.481162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078314.673170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078315.185120:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078315.185152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078315.185261:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078315.185264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078315.505152:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078315.697173:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078316.209181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078316.209181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078316.209187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078316.209289:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078316.529137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078316.529145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078316.529149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078316.529155:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078316.529163:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4022 00000100:00080000:1.0:1769078316.529174:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078316.529182:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078316.529224:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913620224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078316.529254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078316.529259:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078316.529260:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078316.529264:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4022 00000100:00080000:1.0:1769078316.529267:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078316.529270:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078316.529279:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078316.529289:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913620352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078316.529296:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078316.529299:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078316.529301:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078316.529304:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4022 00000100:00080000:1.0:1769078316.529306:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078316.529309:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078316.529312:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078316.529319:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913620224:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078316.529322:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913620480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078316.529335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078316.529339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078316.529341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078316.529345:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4022 00000100:00080000:1.0:1769078316.529349:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078316.529360:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078316.529366:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078316.529379:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913620608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078316.529388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078316.529391:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078316.529393:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078316.529397:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4022 00000100:00080000:1.0:1769078316.529400:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078316.529402:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078316.529406:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078316.529416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913620736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078316.529422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078316.529425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078316.529426:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078316.529428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4022 00000100:00080000:1.0:1769078316.529430:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078316.529433:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078316.529436:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078316.529444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913620864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078316.529447:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913620352:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078316.529454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078316.529457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078316.529459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078316.529462:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4022 00000100:00080000:1.0:1769078316.529465:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078316.529467:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078316.529470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078316.529479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913620992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078316.529507:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913620480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078316.529532:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913620608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078316.529556:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078316.529578:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078316.529609:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078316.721178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078317.233116:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078317.233123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078317.233145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078317.233167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078317.553193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078317.745155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078318.257234:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078318.257235:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078318.257236:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078318.257282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078318.577243:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078318.769186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078319.281138:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078319.281173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078319.281176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078319.281236:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078319.601330:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769078319.601465:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769078319.601520:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078319.601561:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078319.601611:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078319.601620:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078319.601628:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078319.601635:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078319.601644:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078319.601657:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078319.602004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078316/real 1769078319] req@ffff9350100c9c00 x1855008913620480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078319.602039:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913620480/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078319.602075:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078319.602081:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078319.602086:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913620480:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078319.602095:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078316/real 1769078319] req@ffff9350100cb100 x1855008913620608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078319.602109:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913620608/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078319.602118:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078319.602121:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078319.602125:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913620608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078319.602132:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078316/real 1769078319] req@ffff9350100c9180 x1855008913620736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078319.602147:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913620736/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078319.602157:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078319.602160:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078319.602163:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620736:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078319.602170:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078316/real 1769078319] req@ffff9350100c8a80 x1855008913620864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078319.602182:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913620864/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078319.602190:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078319.602194:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078319.602197:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078319.602204:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078316/real 1769078319] req@ffff9350100c8000 x1855008913620992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078319.602219:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913620992/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078319.602227:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078319.602231:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078319.602249:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913620992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078319.602258:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078316/real 1769078319] req@ffff9350100c9880 x1855008913620224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078319.602270:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913620224/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078319.602278:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078319.602281:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078319.602284:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913620224:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078319.602291:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078316/real 1769078319] req@ffff9350100cad80 x1855008913620352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078319.602303:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913620352/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078319.602310:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078319.602314:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078319.602317:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913620352:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769078319.793180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078320.305136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078320.305136:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078320.305136:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078320.305191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078320.625244:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078320.817171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078321.329169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078321.329170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078321.329170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078321.329200:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078321.649180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078321.649194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078321.649198:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078321.649210:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4027 00000100:00100000:0.0:1769078321.649216:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078321.649224:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078321.649233:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078321.649278:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913621120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078321.649309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078321.649312:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078321.649314:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078321.649318:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4027 00000100:00080000:3.0:1769078321.649321:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078321.649326:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078321.649338:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078321.649353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913621248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078321.649355:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913621120:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078321.649371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078321.649374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078321.649377:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078321.649381:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4027 00000100:00080000:3.0:1769078321.649385:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078321.649389:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078321.649394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078321.649407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913621376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078321.649415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078321.649418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078321.649420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078321.649423:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4027 00000100:00080000:3.0:1769078321.649437:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078321.649441:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078321.649446:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078321.649447:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078321.649458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913621504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078321.649467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078321.649471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078321.649473:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078321.649477:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4027 00000100:00080000:3.0:1769078321.649480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078321.649484:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078321.649489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078321.649500:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078321/real 1769078321] req@ffff935002a7e300 x1855008913621120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078321.649502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913621632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078321.649529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078321.649533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078321.649535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078321.649541:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913621120/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078321.649541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4027 00000100:00080000:3.0:1769078321.649549:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078321.649556:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078321.649560:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769078321.649562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078321.649570:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078321.649575:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913621120:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078321.649583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913621760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078321.649597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078321.649599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078321.649601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078321.649607:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4027 00000100:00100000:2.0:1769078321.649608:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913621248:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078321.649611:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078321.649618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078321.649624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078321.649636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913621888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078321.649645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078321.649662:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078321/real 1769078321] req@ffff935002a7d180 x1855008913621248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078321.649676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913621248/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078321.649686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078321.649689:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078321.649693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913621248:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078321.649699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913621376:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078321.649732:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078321.649743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078321/real 1769078321] req@ffff935002a7dc00 x1855008913621376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078321.649754:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913621376/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078321.649761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078321.649764:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078321.649767:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913621376:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078321.649772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913621504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078321.649790:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078321.649800:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078321/real 1769078321] req@ffff935002a7c380 x1855008913621504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078321.649817:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913621504/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078321.649824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078321.649827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078321.649829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913621504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078321.649835:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913621632:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078321.649851:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078321.649860:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078321/real 1769078321] req@ffff935002a7fb80 x1855008913621632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078321.649872:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913621632/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078321.649880:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078321.649912:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078321.649916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913621632:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078321.649922:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913621760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078321.649939:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078321.649947:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078321/real 1769078321] req@ffff935002a7c700 x1855008913621760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078321.649955:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913621760/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078321.649968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078321.649971:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078321.649974:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913621760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078321.649997:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913621888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078321.650017:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078321.650027:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078321/real 1769078321] req@ffff935002a7f100 x1855008913621888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078321.650038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913621888/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078321.650045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078321.650047:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078321.650080:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913621888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078321.841166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078322.353163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078322.353163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078322.353163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078322.353231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078322.673262:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078322.865195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078323.377172:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078323.377173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078323.377173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078323.377226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078323.697193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078323.889178:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078324.401164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078324.401284:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078324.401292:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078324.401368:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078324.721186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078324.913182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078325.425183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078325.425204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078325.425229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078325.425257:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078325.745154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078325.937147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078326.449153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078326.449199:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078326.449207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078326.449212:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078326.769180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078326.769298:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078326.769304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078326.769315:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078326.769325:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4032 00000100:00080000:0.0:1769078326.769336:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078326.769343:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078326.769383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913622016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078326.769411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078326.769415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078326.769417:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078326.769421:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4032 00000100:00080000:0.0:1769078326.769432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078326.769438:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078326.769449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078326.769464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913622144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078326.769480:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913622016:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078326.769486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078326.769498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078326.769500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078326.769505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4032 00000100:00080000:0.0:1769078326.769510:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078326.769515:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078326.769520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078326.769545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913622272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078326.769552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078326.769555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078326.769556:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078326.769560:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4032 00000100:00080000:0.0:1769078326.769563:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078326.769566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078326.769570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078326.769572:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078326.769583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913622400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078326.769593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078326.769596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078326.769597:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078326.769601:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4032 00000100:00080000:0.0:1769078326.769605:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078326.769608:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078326.769612:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078326.769621:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913622528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078326.769628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078326.769632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078326.769635:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078326.769635:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078326/real 1769078326] req@ffff9350103f9c00 x1855008913622016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078326.769645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4032 00000100:00080000:0.0:1769078326.769651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078326.769662:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078326.769672:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078326.769683:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913622016/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078326.769696:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913622656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078326.769699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078326.769710:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769078326.769722:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078326.769725:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913622016:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078326.769732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078326.769744:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078326.769750:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4032 00000100:00080000:0.0:1769078326.769756:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078326.769763:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913622144:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078326.769764:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078326.769775:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078326.769790:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913622784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078326.769819:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078326.769842:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078326/real 1769078326] req@ffff9350103f9180 x1855008913622144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078326.769862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913622144/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078326.769869:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078326.769872:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078326.769874:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913622144:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078326.769879:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913622272:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078326.769895:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078326.769901:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078326/real 1769078326] req@ffff9350103f9f80 x1855008913622272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078326.769908:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913622272/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078326.769913:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078326.769915:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078326.769917:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913622272:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078326.769921:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913622400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078326.769929:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078326.769935:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078326/real 1769078326] req@ffff9350103f9500 x1855008913622400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078326.769941:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913622400/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078326.769946:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078326.769948:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078326.769949:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913622400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078326.769953:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913622528:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078326.769969:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078326.769978:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078326/real 1769078326] req@ffff9350103fbb80 x1855008913622528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078326.769989:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913622528/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078326.769998:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078326.770001:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078326.770004:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913622528:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078326.770011:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913622656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078326.770025:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078326.770031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078326/real 1769078326] req@ffff9350103f8380 x1855008913622656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078326.770038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913622656/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078326.770043:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078326.770046:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078326.770075:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913622656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078326.770092:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913622784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078326.770103:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078326.770118:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078326/real 1769078326] req@ffff9350103fb100 x1855008913622784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078326.770125:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913622784/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078326.770130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078326.770132:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078326.770134:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913622784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078326.961206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078327.473192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078327.473192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078327.473193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078327.473244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078327.793260:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078327.985116:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078328.497167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078328.497167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078328.497167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078328.497204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078328.817236:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078329.009179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078329.521156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078329.521156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078329.521164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078329.521228:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078329.841186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078330.033128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078330.545186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078330.545187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078330.545188:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078330.545287:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078330.865194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078331.057152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078331.569197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078331.569197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078331.569212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078331.569238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078331.889191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.889214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078331.889219:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078331.889223:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078331.889232:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4037 00000100:00080000:3.0:1769078331.889246:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.889254:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078331.889303:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913622912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.889334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.889338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078331.889340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.889344:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4037 00000100:00080000:3.0:1769078331.889347:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.889351:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078331.889363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078331.889377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913623040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078331.889382:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913622912:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078331.889395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.889399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078331.889400:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.889406:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4037 00000100:00080000:3.0:1769078331.889409:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.889412:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078331.889417:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078331.889428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913623168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.889436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.889439:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078331.889441:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.889444:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4037 00000100:00080000:3.0:1769078331.889450:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.889453:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078331.889457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078331.889463:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078331.889476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913623296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.889486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.889489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078331.889491:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.889495:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4037 00000100:00080000:3.0:1769078331.889500:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.889504:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078331.889509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078331.889515:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff935002a7df80 x1855008913622912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078331.889523:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913623424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.889536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.889540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769078331.889543:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913622912/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078331.889544:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.889553:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4037 00000100:00080000:3.0:1769078331.889559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078331.889565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078331.889565:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078331.889571:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:3.0:1769078331.889574:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078331.889575:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913622912:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078331.889589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913623552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.889599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.889602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078331.889603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.889607:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4037 00000100:00080000:3.0:1769078331.889609:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078331.889614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623040:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078331.889615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078331.889620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078331.889633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913623680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078331.889651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078331.889664:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff935002a7f800 x1855008913623040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078331.889673:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913623040/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078331.889679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078331.889681:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078331.889683:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623040:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078331.889688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623168:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078331.889706:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078331.889728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff935002a7c000 x1855008913623168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078331.889741:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913623168/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078331.889748:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078331.889752:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078331.889754:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623168:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078331.889759:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078331.889773:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078331.889780:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff935002a7ca80 x1855008913623296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078331.889787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913623296/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078331.889793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078331.889795:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078331.889797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078331.889800:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913623424:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078331.889810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078331.889814:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff935002a7f100 x1855008913623424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078331.889821:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913623424/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078331.889826:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078331.889827:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078331.889829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913623424:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078331.889832:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913623552:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078331.889840:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078331.889845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff935002a7c700 x1855008913623552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078331.889859:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913623552/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078331.889867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078331.889868:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078331.889870:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913623552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078331.889885:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913623680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078331.889895:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078331.889901:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff935002a7fb80 x1855008913623680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078331.889907:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913623680/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078331.889912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078331.889914:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078331.889916:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913623680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078332.081184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078332.593167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078332.593167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078332.593167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078332.593243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078332.913193:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078333.105188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078333.617161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078333.617174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078333.617175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078333.617231:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078333.937230:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078334.129186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078334.641168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078334.641168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078334.641177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078334.641204:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078334.961194:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078335.153167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078335.665215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078335.665265:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078335.665350:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078335.665364:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078335.985130:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078336.177164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078336.689152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078336.689177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078336.689206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078336.689237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078337.009161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078337.009163:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078337.009171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078337.009175:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078337.009182:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4042 00000100:00080000:3.0:1769078337.009191:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078337.009197:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078337.009226:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913623808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078337.009249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078337.009252:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078337.009253:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078337.009256:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4042 00000100:00080000:3.0:1769078337.009258:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078337.009260:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078337.009267:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078337.009274:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913623936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078337.009281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078337.009283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078337.009284:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078337.009287:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4042 00000100:00080000:3.0:1769078337.009288:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078337.009292:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078337.009295:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078337.009300:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913624064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078337.009310:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913623808:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078337.009311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078337.009313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078337.009314:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078337.009317:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4042 00000100:00080000:3.0:1769078337.009326:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078337.009333:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078337.009336:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078337.009345:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913624192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078337.009351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078337.009353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078337.009354:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078337.009357:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4042 00000100:00080000:3.0:1769078337.009359:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078337.009361:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078337.009363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078337.009372:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913624320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078337.009377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078337.009379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078337.009379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078337.009381:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4042 00000100:00080000:3.0:1769078337.009393:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078337.009396:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078337.009399:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078337.009409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913624448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078337.009411:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078337.009428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078337.009430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078337.009431:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078337.009434:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4042 00000100:00080000:3.0:1769078337.009437:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078337.009439:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078337.009443:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078337.009452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913624576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078337.009479:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078337/real 1769078337] req@ffff935002a7c380 x1855008913623808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078337.009495:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913623808/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078337.009510:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078337.009514:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078337.009517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913623808:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078337.009523:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623936:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078337.009541:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078337.009550:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078337/real 1769078337] req@ffff935002a7dc00 x1855008913623936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078337.009557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913623936/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078337.009564:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078337.009566:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078337.009568:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913623936:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078337.009589:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624064:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078337.009600:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078337.009605:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078337/real 1769078337] req@ffff935002a7d180 x1855008913624064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078337.009612:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913624064/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078337.009617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078337.009619:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078337.009621:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624064:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078337.009640:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078337.009654:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078337.009667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078337/real 1769078337] req@ffff935002a7e300 x1855008913624192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078337.009674:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913624192/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078337.009680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078337.009682:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078337.009684:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078337.009688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913624320:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078337.009704:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078337.009713:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078337/real 1769078337] req@ffff935002a7fb80 x1855008913624320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078337.009724:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913624320/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078337.009733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078337.009736:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078337.009739:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913624320:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078337.009745:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913624448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078337.009764:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078337.009774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078337/real 1769078337] req@ffff935002a7c700 x1855008913624448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078337.009787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913624448/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078337.009795:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078337.009798:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078337.009801:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913624448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078337.009807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913624576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078337.009821:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078337.009827:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078337/real 1769078337] req@ffff935002a7f100 x1855008913624576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078337.009835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913624576/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078337.009842:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078337.009846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078337.009849:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913624576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078337.201144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078337.713109:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078337.713111:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078337.713153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078337.713267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078338.033150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078338.225164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078338.737118:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078338.737120:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078338.737161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078338.737213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078339.057144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078339.249171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078339.761144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078339.761153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078339.761173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078339.761233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078340.081164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078340.273179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078340.785109:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078340.785110:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078340.785120:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078340.785201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078341.105170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078341.297140:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078341.809174:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078341.809174:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078341.809174:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078341.809237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078342.129178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078342.129192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078342.129200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078342.129205:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078342.129216:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4047 00000100:00080000:3.0:1769078342.129228:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078342.129235:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078342.129276:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913624704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078342.129305:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078342.129308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078342.129310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078342.129314:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4047 00000100:00080000:3.0:1769078342.129317:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078342.129321:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078342.129332:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078342.129343:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913624832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078342.129352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078342.129355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078342.129357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078342.129361:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4047 00000100:00080000:3.0:1769078342.129366:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078342.129369:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078342.129374:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078342.129383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913624960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078342.129394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078342.129399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078342.129403:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078342.129405:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913624704:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078342.129410:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4047 00000100:00080000:3.0:1769078342.129420:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078342.129426:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078342.129431:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078342.129443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913625088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078342.129459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078342.129463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078342.129465:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078342.129469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4047 00000100:00080000:3.0:1769078342.129481:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078342.129485:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078342.129490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078342.129491:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078342.129511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913625216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078342.129523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078342.129526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078342.129528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078342.129532:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4047 00000100:00080000:3.0:1769078342.129536:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078342.129553:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078342.129558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078342.129565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913625344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078342.129572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078342.129577:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078342/real 1769078342] req@ffff935002a7ca80 x1855008913624704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078342.129578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078342.129583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078342.129597:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4047 00000100:00080000:3.0:1769078342.129603:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078342.129609:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078342.129614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078342.129620:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913624704/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078342.129633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913625472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078342.129692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078342.129700:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078342.129706:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913624704:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078342.129714:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624832:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078342.129740:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078342.129754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078342/real 1769078342] req@ffff935002a7c000 x1855008913624832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078342.129776:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913624832/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078342.129786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078342.129789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078342.129793:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624832:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078342.129818:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624960:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078342.129840:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078342.129850:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078342/real 1769078342] req@ffff935002a7f800 x1855008913624960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078342.129862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913624960/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078342.129870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078342.129874:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078342.129877:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913624960:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078342.129884:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078342.129902:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078342.129914:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078342/real 1769078342] req@ffff935002a7df80 x1855008913625088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078342.129925:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913625088/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078342.129934:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078342.129937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078342.129940:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078342.129946:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913625216:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078342.129965:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078342.129975:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078342/real 1769078342] req@ffff935002a7f100 x1855008913625216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078342.129986:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913625216/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078342.129995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078342.129998:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078342.130001:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913625216:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078342.130008:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913625344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078342.130027:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078342.130038:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078342/real 1769078342] req@ffff935002a7c700 x1855008913625344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078342.130091:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913625344/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078342.130100:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078342.130103:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078342.130107:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913625344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078342.130113:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913625472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078342.130132:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078342.130143:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078342/real 1769078342] req@ffff935002a7fb80 x1855008913625472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078342.130154:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913625472/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078342.130162:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078342.130165:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078342.130168:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913625472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078342.321200:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078342.833191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078342.833191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078342.833191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078342.833282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078343.153177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078343.345203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078343.857160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078343.857161:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078343.857161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078343.857243:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078344.177177:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078344.369194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078344.881188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078344.881189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078344.881210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078344.881270:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078345.201186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078345.393175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078345.905156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078345.905156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078345.905181:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078345.905303:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078346.225203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078346.417211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078346.929183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078346.929187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078346.929218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078346.929237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:3.0:1769078347.249189:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078347.249199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078347.249204:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078347.249215:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4052 00000100:00100000:0.0:1769078347.249219:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078347.249226:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078347.249234:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078347.249277:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913625600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078347.249297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078347.249300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078347.249301:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078347.249304:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4052 00000100:00080000:3.0:1769078347.249306:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078347.249309:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078347.249318:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078347.249325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913625728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078347.249332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078347.249340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078347.249344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078347.249349:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4052 00000100:00080000:3.0:1769078347.249352:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078347.249355:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078347.249362:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078347.249374:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913625856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078347.249383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078347.249387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078347.249388:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078347.249392:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4052 00000100:00080000:3.0:1769078347.249395:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078347.249399:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078347.249404:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078347.249413:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913625600:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078347.249420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913625984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078347.249443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078347.249446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078347.249448:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078347.249466:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4052 00000100:00080000:3.0:1769078347.249469:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078347.249473:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078347.249477:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078347.249490:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913626112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078347.249495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078347.249498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078347.249499:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078347.249502:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4052 00000100:00080000:3.0:1769078347.249504:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078347.249506:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078347.249510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078347.249516:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078347.249520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913626240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078347.249527:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078347.249529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078347.249530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078347.249534:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4052 00000100:00080000:3.0:1769078347.249538:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078347.249543:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078347.249546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078347.249552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913626368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078347.249583:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078347/real 1769078347] req@ffff935002a7e300 x1855008913625600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078347.249602:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913625600/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078347.249635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078347.249640:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078347.249644:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913625600:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078347.249652:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625728:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078347.249675:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078347.249687:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078347/real 1769078347] req@ffff935002a7d180 x1855008913625728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078347.249698:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913625728/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078347.249724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078347.249761:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078347.249766:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625728:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078347.249790:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625856:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078347.249909:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078347.249939:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078347/real 1769078347] req@ffff935002a7dc00 x1855008913625856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078347.249952:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913625856/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078347.249961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078347.249964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078347.249968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625856:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078347.249996:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078347.250133:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078347.250191:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078347/real 1769078347] req@ffff935002a7c380 x1855008913625984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078347.250243:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913625984/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078347.250252:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078347.250256:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078347.250273:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913625984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078347.250280:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913626112:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078347.250301:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078347.250349:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078347/real 1769078347] req@ffff935002a7fb80 x1855008913626112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078347.250397:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913626112/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078347.250406:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078347.250409:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078347.250412:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913626112:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078347.250419:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913626240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078347.250439:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078347.250463:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078347/real 1769078347] req@ffff935002a7c700 x1855008913626240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078347.250477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913626240/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078347.250484:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078347.250487:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078347.250490:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913626240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078347.250496:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913626368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078347.250516:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078347.250548:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078347/real 1769078347] req@ffff935002a7f100 x1855008913626368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078347.250573:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913626368/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078347.250581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078347.250584:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078347.250587:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913626368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078347.441121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078347.953209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078347.953211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078347.953222:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078347.953258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078348.273204:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078348.465204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078348.977163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078348.977163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078348.977192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078348.977219:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078349.297190:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078349.489123:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078350.001148:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078350.001149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078350.001179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078350.001193:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078350.321172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078350.513207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078351.025195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078351.025214:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078351.025266:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078351.025293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078351.345169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078351.537110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078352.049126:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078352.049136:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078352.049159:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078352.049180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078352.369176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078352.369176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078352.369185:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078352.369189:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078352.369197:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4058 00000100:00080000:2.0:1769078352.369216:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078352.369224:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078352.369260:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913626496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078352.369283:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078352.369286:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078352.369288:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078352.369292:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4058 00000100:00080000:2.0:1769078352.369295:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078352.369299:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078352.369309:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078352.369321:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913626624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078352.369324:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913626496:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078352.369330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078352.369333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078352.369335:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078352.369338:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4058 00000100:00080000:2.0:1769078352.369341:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078352.369344:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078352.369348:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078352.369355:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913626752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078352.369363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078352.369390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078352.369392:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078352.369396:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4058 00000100:00080000:2.0:1769078352.369399:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078352.369402:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:3.0:1769078352.369404:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769078352.369406:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078352.369415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913626880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078352.369423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078352.369425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078352.369427:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078352.369437:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4058 00000100:00100000:3.0:1769078352.369438:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078352/real 1769078352] req@ffff935002a62680 x1855008913626496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078352.369440:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078352.369444:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078352.369448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078352.369455:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913626496/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078352.369458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913627008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078352.369465:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078352.369469:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769078352.369472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769078352.369473:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913626496:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078352.369475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078352.369477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078352.369480:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4058 00000100:00080000:2.0:1769078352.369482:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078352.369485:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078352.369489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078352.369498:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913627136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078352.369500:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913626624:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769078352.369507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078352.369510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078352.369512:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078352.369514:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4058 00000100:00080000:2.0:1769078352.369517:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769078352.369519:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078352.369520:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078352.369524:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078352.369531:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078352/real 1769078352] req@ffff935002a61180 x1855008913626624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078352.369532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913627264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078352.369543:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913626624/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078352.369552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078352.369555:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078352.369558:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913626624:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078352.369564:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913626752:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078352.369582:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078352.369591:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078352/real 1769078352] req@ffff935002a61880 x1855008913626752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078352.369598:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913626752/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078352.369603:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078352.369605:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078352.369607:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913626752:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078352.369610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913626880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078352.369620:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078352.369625:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078352/real 1769078352] req@ffff935002a62300 x1855008913626880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078352.369631:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913626880/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078352.369636:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078352.369638:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078352.369640:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913626880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078352.369643:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627008:192.168.204.121@tcp:38:kworker.0 00000800:00000100:3.0:1769078352.369650:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078352.369655:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078352/real 1769078352] req@ffff935002a60a80 x1855008913627008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078352.369671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913627008/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078352.369678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078352.369682:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078352.369684:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627008:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078352.369698:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078352.369714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078352.369723:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078352/real 1769078352] req@ffff935002a61f80 x1855008913627136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078352.369733:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913627136/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078352.369739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078352.369742:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078352.369745:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078352.369750:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:3.0:1769078352.369765:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078352.369774:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078352/real 1769078352] req@ffff935002a61c00 x1855008913627264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078352.369783:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913627264/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078352.369789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078352.369791:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078352.369794:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078352.561147:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078353.073131:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078353.073132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078353.073163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078353.073171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078353.393131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078353.585160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078354.097143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078354.097143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078354.097170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078354.097171:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078354.417162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078354.609137:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078355.121134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078355.121134:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078355.121141:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078355.121163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078355.441120:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078355.633152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078356.145125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078356.145167:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078356.145172:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078356.145177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078356.465207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078356.657155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078357.169159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078357.169159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078357.169223:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078357.169261:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078357.489128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078357.489255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078357.489263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078357.489267:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078357.489276:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4063 00000100:00080000:3.0:1769078357.489297:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078357.489310:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078357.489353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913627392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078357.489378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078357.489381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078357.489382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078357.489386:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4063 00000100:00080000:3.0:1769078357.489388:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078357.489391:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078357.489400:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078357.489414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913627520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078357.489421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078357.489425:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913627392:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078357.489425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078357.489428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078357.489434:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4063 00000100:00080000:3.0:1769078357.489437:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078357.489440:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078357.489444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078357.489457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913627648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078357.489467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078357.489470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078357.489472:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078357.489476:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4063 00000100:00080000:3.0:1769078357.489490:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078357.489494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078357.489499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078357.489502:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078357.489513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913627776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078357.489524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078357.489528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078357.489529:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078357.489534:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4063 00000100:00080000:3.0:1769078357.489539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078357.489545:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078357/real 1769078357] req@ffff935002a7df80 x1855008913627392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078357.489546:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078357.489561:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078357.489568:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913627392/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078357.489579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913627904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078357.489581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078357.489590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078357.489595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078357.489598:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913627392:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078357.489601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078357.489604:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078357.489609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4063 00000100:00080000:3.0:1769078357.489614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078357.489618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078357.489625:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913627520:192.168.204.121@tcp:38:kworker.0 00000008:00100000:3.0:1769078357.489625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078357.489643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913628032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078357.489656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078357.489666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078357.489670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078357.489672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078357.489676:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078357/real 1769078357] req@ffff935002a7f800 x1855008913627520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078357.489678:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4063 00000100:00080000:3.0:1769078357.489685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078357.489691:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078357.489697:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078357.489698:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913627520/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078357.489709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078357.489713:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:3.0:1769078357.489713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913628160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078357.489716:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913627520:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078357.489727:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913627648:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078357.489746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078357.489754:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078357/real 1769078357] req@ffff935002a7c000 x1855008913627648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078357.489761:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913627648/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078357.489766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078357.489768:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078357.489769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913627648:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078357.489772:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913627776:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078357.489781:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078357.489788:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078357/real 1769078357] req@ffff935002a7ca80 x1855008913627776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078357.489798:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913627776/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078357.489805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078357.489808:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078357.489811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913627776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078357.489817:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627904:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078357.489834:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078357.489844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078357/real 1769078357] req@ffff935002a7f100 x1855008913627904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078357.489854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913627904/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078357.489862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078357.489865:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078357.489867:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913627904:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078357.489885:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078357.489903:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078357.489910:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078357/real 1769078357] req@ffff935002a7c700 x1855008913628032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078357.489918:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913628032/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078357.489923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078357.489924:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078357.489926:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078357.489930:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628160:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078357.489939:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078357.489944:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078357/real 1769078357] req@ffff935002a7fb80 x1855008913628160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078357.489951:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913628160/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078357.489955:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078357.489957:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078357.489959:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078357.681169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078358.193165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078358.193165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078358.194785:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078358.194860:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078358.513172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078358.705152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078359.217202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078359.217211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078359.217212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078359.217259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078359.537200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078359.729171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078360.241179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078360.241179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078360.241179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078360.241226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078360.561155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078360.753158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078361.265153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078361.265153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078361.265156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078361.265187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078361.585456:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078361.777169:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078362.289182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078362.289182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078362.289182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078362.289275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078362.609178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078362.609183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078362.609190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078362.609211:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078362.609223:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4068 00000100:00080000:3.0:1769078362.609235:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078362.609253:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078362.609302:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913628288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078362.609446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078362.609451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078362.609453:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078362.609459:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4068 00000100:00080000:3.0:1769078362.609464:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078362.609468:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078362.609480:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078362.609493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913628416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078362.609504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078362.609507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078362.609509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078362.609514:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4068 00000100:00080000:3.0:1769078362.609517:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078362.609521:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078362.609525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078362.609534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913628544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078362.609542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078362.609545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078362.609546:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078362.609550:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4068 00000100:00080000:3.0:1769078362.609553:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078362.609556:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078362.609560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078362.609569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913628672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078362.609579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078362.609582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078362.609584:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078362.609587:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4068 00000100:00080000:3.0:1769078362.609590:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078362.609594:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078362.609599:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078362.609611:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913628800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078362.609620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078362.609627:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913628288:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078362.609627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078362.609632:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078362.609637:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4068 00000100:00080000:3.0:1769078362.609639:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078362.609642:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078362.609645:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078362.609656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913628928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078362.609663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078362.609665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078362.609666:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078362.609669:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4068 00000100:00080000:3.0:1769078362.609670:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078362.609673:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078362.609675:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078362.609682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913629056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078362.609797:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078362.609956:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078362/real 1769078362] req@ffff935002a7c380 x1855008913628288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078362.609990:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913628288/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078362.610022:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078362.610027:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078362.610124:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913628288:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078362.610154:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913628416:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078362.610237:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078362.610247:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078362/real 1769078362] req@ffff935002a7dc00 x1855008913628416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078362.610255:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913628416/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078362.610261:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078362.610264:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078362.610266:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913628416:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078362.610270:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913628544:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078362.610283:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078362.610288:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078362/real 1769078362] req@ffff935002a7d180 x1855008913628544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078362.610295:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913628544/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078362.610300:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078362.610302:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078362.610303:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913628544:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078362.610323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913628672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078362.610332:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078362.610337:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078362/real 1769078362] req@ffff935002a7e300 x1855008913628672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078362.610344:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913628672/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078362.610349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078362.610351:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078362.610353:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913628672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078362.610356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628800:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078362.610418:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078362.610472:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078362/real 1769078362] req@ffff935002a7fb80 x1855008913628800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078362.610623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913628800/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078362.610648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078362.610652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078362.610655:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628800:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078362.610687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078362.610717:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078362.610740:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078362/real 1769078362] req@ffff935002a7c700 x1855008913628928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078362.610780:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913628928/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078362.610805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078362.610808:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078362.610811:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913628928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078362.610815:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078362.610845:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078362.610855:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078362/real 1769078362] req@ffff935002a7f100 x1855008913629056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078362.610882:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913629056/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078362.610904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078362.610907:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078362.610920:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078362.801160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078363.313198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078363.313198:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078363.313206:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078363.313254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078363.633221:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078363.825234:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078364.337202:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078364.337202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078364.337207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078364.337259:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078364.657151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078364.849144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078365.361163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078365.361163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078365.361168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078365.361282:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078365.681168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078365.873118:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078366.385182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078366.385182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078366.385185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078366.385235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078366.705151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078366.897110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078367.409194:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078367.409196:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078367.409215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078367.409256:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078367.729161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078367.729182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078367.729191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078367.729196:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078367.729207:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4073 00000100:00080000:3.0:1769078367.729218:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078367.729224:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078367.729273:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913629184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078367.729297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078367.729299:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078367.729301:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078367.729304:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4073 00000100:00080000:3.0:1769078367.729306:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078367.729309:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078367.729319:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078367.729328:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913629312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078367.729336:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078367.729338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078367.729339:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078367.729341:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4073 00000100:00080000:3.0:1769078367.729343:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078367.729348:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078367.729351:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913629184:192.168.204.121@tcp:250:kworker.0 00000008:00100000:3.0:1769078367.729363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078367.729378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913629440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078367.729386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078367.729388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078367.729389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078367.729392:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4073 00000100:00080000:3.0:1769078367.729394:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078367.729397:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078367.729401:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078367.729411:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913629568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078367.729421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078367.729424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078367.729429:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078367.729430:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078367.729436:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4073 00000100:00080000:3.0:1769078367.729441:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078367.729444:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078367.729448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078367.729459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913629696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078367.729465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078367.729468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078367.729469:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078367.729471:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4073 00000100:00080000:3.0:1769078367.729474:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078367.729477:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078367.729480:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078367.729486:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078367/real 1769078367] req@ffff935002a7ca80 x1855008913629184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078367.729489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913629824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078367.729522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078367.729535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078367.729538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078367.729541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4073 00000100:00080000:3.0:1769078367.729546:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078367.729549:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078367.729553:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078367.729555:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913629184/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078367.729563:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913629952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078367.729568:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078367.729577:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078367.729590:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913629184:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078367.729616:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913629312:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078367.729637:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078367.729646:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078367/real 1769078367] req@ffff935002a7c000 x1855008913629312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078367.729655:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913629312/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078367.729661:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078367.729663:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078367.729666:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913629312:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078367.729671:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913629440:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078367.729682:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078367.729688:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078367/real 1769078367] req@ffff935002a7f800 x1855008913629440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078367.729695:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913629440/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078367.729701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078367.729702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078367.729704:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913629440:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078367.729708:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913629568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078367.729718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078367.729724:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078367/real 1769078367] req@ffff935002a7df80 x1855008913629568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078367.729731:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913629568/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078367.729737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078367.729738:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078367.729740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913629568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078367.729744:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629696:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078367.729753:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078367.729759:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078367/real 1769078367] req@ffff935002a7f100 x1855008913629696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078367.729773:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913629696/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078367.729778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078367.729780:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078367.729782:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629696:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078367.729786:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078367.729797:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078367.729803:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078367/real 1769078367] req@ffff935002a7c700 x1855008913629824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078367.729815:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913629824/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078367.729821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078367.729824:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078367.729826:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078367.729829:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078367.729839:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078367.729845:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078367/real 1769078367] req@ffff935002a7fb80 x1855008913629952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078367.729852:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913629952/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078367.729858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078367.729859:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078367.729861:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913629952:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078367.921146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078368.433215:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078368.433216:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078368.433388:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078368.433411:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078368.753174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078368.945212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078369.457192:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078369.457228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078369.457285:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078369.457297:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078369.777218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078369.969223:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078370.481226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078370.481230:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078370.481231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078370.481264:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078370.801213:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078370.993234:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078371.505182:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078371.505182:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078371.505182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078371.505253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078371.825160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078372.017201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078372.529173:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078372.529173:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078372.529173:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078372.529220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078372.849160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078372.849351:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.849358:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078372.849362:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.849373:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4078 00000100:00080000:0.0:1769078372.849386:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.849394:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078372.849433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913630080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.849457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.849460:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078372.849461:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.849464:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4078 00000100:00080000:0.0:1769078372.849466:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.849468:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078372.849477:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078372.849484:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913630208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.849491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.849493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078372.849494:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.849497:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4078 00000100:00080000:0.0:1769078372.849499:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.849501:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078372.849505:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078372.849510:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913630336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.849517:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.849520:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078372.849522:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078372.849522:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913630080:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078372.849527:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4078 00000100:00080000:0.0:1769078372.849532:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.849534:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078372.849536:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078372.849549:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913630464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.849557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.849560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078372.849561:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.849565:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4078 00000100:00080000:0.0:1769078372.849569:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.849573:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078372.849578:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078372.849588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913630592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.849597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.849600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769078372.849605:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078372.849612:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.849618:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4078 00000100:00080000:0.0:1769078372.849624:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.849628:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078372.849633:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078372.849647:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913630720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.849656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078372.849659:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078372/real 1769078372] req@ffff9350103faa00 x1855008913630080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078372.849660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078372.849663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.849671:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4078 00000100:00080000:0.0:1769078372.849677:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.849681:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078372.849686:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078372.849690:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913630080/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078372.849701:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913630848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078372.849705:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.849712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078372.849718:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913630080:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078372.849801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913630208:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078372.849827:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078372.849841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078372/real 1769078372] req@ffff9350103f8700 x1855008913630208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.849854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913630208/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.849863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.849867:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078372.849870:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913630208:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078372.849897:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913630336:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078372.849916:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078372.849926:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078372/real 1769078372] req@ffff9350103fb800 x1855008913630336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.849937:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913630336/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.849945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.849948:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078372.849952:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913630336:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078372.849958:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913630464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078372.849974:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078372.849984:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078372/real 1769078372] req@ffff9350103fb480 x1855008913630464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.849996:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913630464/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.850004:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.850007:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078372.850010:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913630464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078372.850016:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913630592:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078372.850093:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078372.850105:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078372/real 1769078372] req@ffff9350103fad80 x1855008913630592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.850117:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913630592/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.850125:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.850129:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078372.850132:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913630592:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078372.850153:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913630720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078372.850174:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078372.850185:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078372/real 1769078372] req@ffff9350103fb100 x1855008913630720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.850201:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913630720/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.850215:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.850219:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078372.850222:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913630720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078372.850228:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913630848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078372.850246:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078372.850255:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078372/real 1769078372] req@ffff9350103f8380 x1855008913630848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.850267:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913630848/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.850275:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.850278:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078372.850281:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913630848:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078373.041189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078373.553197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078373.553199:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078373.553275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078373.553350:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078373.873774:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078374.065159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078374.577175:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078374.577175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078374.577175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078374.577207:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078374.897226:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078375.089211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078375.601186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078375.601187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078375.601187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078375.601402:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078375.921183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078376.113201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078376.625169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078376.625183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078376.625189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078376.625234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078376.945150:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078377.137158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078377.649109:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078377.649125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078377.649132:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078377.649184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078377.969185:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078377.969198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078377.969206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078377.969212:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078377.969224:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4083 00000100:00080000:1.0:1769078377.969236:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078377.969244:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078377.969292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913630976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078377.969322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078377.969327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078377.969329:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078377.969335:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4083 00000100:00080000:1.0:1769078377.969339:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078377.969344:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078377.969357:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078377.969377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913631104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078377.969380:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913630976:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078377.969407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078377.969412:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078377.969414:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078377.969421:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4083 00000100:00080000:1.0:1769078377.969426:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078377.969430:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078377.969438:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078377.969452:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913631232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078377.969463:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078377.969467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078377.969469:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078377.969474:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4083 00000100:00080000:1.0:1769078377.969478:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078377.969482:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078377.969488:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078377.969494:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078377.969508:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913631360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078377.969519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078377.969522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078377.969525:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078377.969530:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4083 00000100:00080000:1.0:1769078377.969536:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078377.969542:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff9350100c9f80 x1855008913630976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078377.969547:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078377.969557:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078377.969576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913631488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078377.969584:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913630976/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078377.969592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078377.969603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078377.969605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078377.969615:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4083 00000100:00080000:2.0:1769078377.969615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078377.969629:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078377.969633:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078377.969639:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078377.969643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913630976:192.168.204.121@tcp:250:kworker.0 00000008:00100000:1.0:1769078377.969649:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078377.969667:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913631616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078377.969676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078377.969680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078377.969682:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078377.969688:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913631104:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078377.969700:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4083 00000100:00080000:1.0:1769078377.969707:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078377.969713:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078377.969718:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078377.969719:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:1.0:1769078377.969739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913631744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078377.969743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff9350100cad80 x1855008913631104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.969787:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913631104/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.969805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.969812:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078377.969816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913631104:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078377.969823:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913631232:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078377.969842:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078377.969852:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff9350100c9880 x1855008913631232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.969862:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913631232/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.969870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.969873:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078377.969876:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913631232:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078377.969881:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913631360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078377.969896:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078377.969904:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff9350100c8000 x1855008913631360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.969915:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913631360/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.969921:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.969924:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078377.969927:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913631360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078377.969933:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913631488:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078377.969962:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078377.969971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff9350100c8a80 x1855008913631488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.969982:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913631488/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.969989:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.969992:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078377.969995:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913631488:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078377.970000:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913631616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078377.970015:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078377.970024:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff9350100c9180 x1855008913631616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.970033:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913631616/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.970045:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.970087:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078377.970090:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913631616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078377.970108:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913631744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078377.970125:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078377.970134:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff9350100cb100 x1855008913631744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.970144:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913631744/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.970152:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.970155:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078377.970158:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913631744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078378.161207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078378.673193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078378.673193:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078378.676262:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078378.676461:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078378.993183:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078379.185180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078379.697275:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078379.698080:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078379.701850:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078379.701851:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078380.017162:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078380.209181:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078380.721162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078380.721162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078380.721179:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078380.721214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078381.041144:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078381.233143:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078381.745142:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078381.745142:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078381.745154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078381.745172:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078382.065151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078382.257153:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078382.769145:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078382.769145:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078382.769145:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078382.769244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078383.089119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078383.089153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078383.089161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078383.089165:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078383.089175:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4088 00000100:00080000:0.0:1769078383.089185:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078383.089191:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078383.089228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913631872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078383.089254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078383.089258:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078383.089260:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078383.089264:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4088 00000100:00080000:0.0:1769078383.089267:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078383.089271:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078383.089280:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078383.089294:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913632000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078383.089304:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078383.089308:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078383.089310:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078383.089315:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4088 00000100:00080000:0.0:1769078383.089318:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078383.089322:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078383.089328:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078383.089335:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913631872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078383.089344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913632128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078383.089357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078383.089360:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078383.089363:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078383.089378:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4088 00000100:00080000:0.0:1769078383.089383:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078383.089387:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078383.089394:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078383.089407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913632256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078383.089417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078383.089420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078383.089423:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078383.089434:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4088 00000100:00080000:0.0:1769078383.089438:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078383.089443:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078383.089448:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078383.089463:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913632384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078383.089475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078383.089479:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632000:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078383.089481:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078383.089486:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078383.089497:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4088 00000100:00080000:0.0:1769078383.089506:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078383.089512:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078383.089519:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078383.089528:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078383.089549:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913632512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078383.089553:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632256:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769078383.089575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078383.089579:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913632384:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078383.089588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078383.089591:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078383.089600:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4088 00000100:00100000:2.0:1769078383.089601:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913632512:192.168.204.121@tcp:8:kworker.0 00000100:00080000:0.0:1769078383.089610:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078383.089618:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078383.089626:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078383.089641:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913632640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078383.089692:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913632640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078383.281158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078383.793165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078383.793165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078383.793181:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078383.793258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078384.113274:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078384.305193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078384.817175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078384.817178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078384.817191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078384.817255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078385.137172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078385.329126:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078385.841163:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078385.841163:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078385.841163:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078385.841198:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078386.161131:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769078386.161319:0:2349:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769078386.161356:0:2349:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078386.161371:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078386.161413:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078386.161425:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078386.161444:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078386.161460:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078383/real 1769078386] req@ffff9350103f9f80 x1855008913632128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769078386.161463:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078386.161493:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078386.161513:0:2349:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078386.161519:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913632128/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078386.161535:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078386.161542:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078386.161547:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632128:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078386.161556:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078383/real 1769078386] req@ffff9350103f9180 x1855008913632256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078386.161571:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913632256/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078386.161579:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078386.161583:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078386.161586:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078386.161593:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078383/real 1769078386] req@ffff9350103f9c00 x1855008913632384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078386.161607:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913632384/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078386.161614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078386.161618:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078386.161620:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913632384:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078386.161627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078383/real 1769078386] req@ffff9350103fbb80 x1855008913631872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078386.161640:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913631872/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078386.161647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078386.161652:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078386.161656:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913631872:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078386.161663:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078383/real 1769078386] req@ffff9350103f9500 x1855008913632000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078386.161676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913632000/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078386.161685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078386.161690:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078386.161699:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632000:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078386.161764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078383/real 1769078386] req@ffff9350103fb100 x1855008913632640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078386.161777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913632640/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078386.161784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078386.161788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078386.161790:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913632640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078386.161810:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078383/real 1769078386] req@ffff9350103f8380 x1855008913632512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078386.161821:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913632512/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078386.161828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078386.161833:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078386.161836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913632512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078386.353159:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078386.865153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078386.865153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078386.865153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078386.865220:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078387.185124:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078387.377151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078387.889153:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078387.889157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078387.889158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078387.889237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078388.209106:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078388.209128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078388.209136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078388.209140:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078388.209151:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4093 00000100:00080000:3.0:1769078388.209163:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078388.209170:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078388.209205:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913632768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078388.209234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078388.209238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078388.209240:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078388.209245:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4093 00000100:00080000:3.0:1769078388.209249:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078388.209254:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078388.209265:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078388.209280:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913632896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078388.209292:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913632768:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078388.209301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078388.209307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078388.209309:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078388.209316:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4093 00000100:00080000:3.0:1769078388.209322:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078388.209327:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078388.209333:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078388.209357:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913633024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078388.209367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078388.209371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078388.209374:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078388.209379:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4093 00000800:00000100:2.0:1769078388.209394:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078388.209397:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078388.209407:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078388.209417:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078388.209433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913633152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078388.209446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078388.209452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078388.209454:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078388.209460:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4093 00000100:00100000:2.0:1769078388.209466:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078388/real 1769078388] req@ffff935002a7e300 x1855008913632768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078388.209470:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078388.209485:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078388.209497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078388.209510:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913632768/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078388.209522:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913633280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078388.209536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078388.209550:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078388.209560:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913632768:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078388.209565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078388.209573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078388.209575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078388.209581:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4093 00000100:00080000:3.0:1769078388.209585:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078388.209590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078388.209597:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078388.209610:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913633408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078388.209623:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632896:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078388.209625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078388.209635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078388.209637:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078388.209644:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4093 00000100:00080000:3.0:1769078388.209653:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078388.209659:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078388.209666:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078388.209667:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078388.209692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913633536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078388.209694:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078388/real 1769078388] req@ffff935002a7d180 x1855008913632896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078388.209715:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913632896/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078388.209726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078388.209730:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078388.209734:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913632896:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078388.209742:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633024:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078388.209781:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078388.209797:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078388/real 1769078388] req@ffff935002a7dc00 x1855008913633024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078388.209812:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913633024/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078388.209822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078388.209826:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078388.209829:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633024:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078388.209837:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078388.209858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078388.209870:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078388/real 1769078388] req@ffff935002a7c380 x1855008913633152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078388.209883:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913633152/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078388.209898:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078388.209901:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078388.209904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078388.209911:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913633280:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078388.209931:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078388.209942:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078388/real 1769078388] req@ffff935002a7fb80 x1855008913633280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078388.209953:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913633280/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078388.209961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078388.209965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078388.209968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913633280:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078388.209993:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913633408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078388.210014:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078388.210027:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078388/real 1769078388] req@ffff935002a7c700 x1855008913633408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078388.210039:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913633408/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078388.210119:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078388.210125:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078388.210129:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913633408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078388.211447:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913633536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078388.211493:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078388.211509:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078388/real 1769078388] req@ffff935002a7f100 x1855008913633536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078388.211521:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913633536/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078388.211533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078388.211537:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078388.211541:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913633536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078388.401171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078388.913188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078388.913215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078388.913264:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078388.913265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078389.233149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078389.425122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078389.937152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078389.937155:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078389.937171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078389.937291:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078390.257112:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078390.449110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078390.961101:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078390.961184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078390.961246:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078390.963871:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078391.281103:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078391.473110:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078391.985128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078391.985149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078391.985162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078391.985178:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078392.305119:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078392.497275:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078393.009160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078393.009167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078393.009180:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078393.009455:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078393.329185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078393.329195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078393.329199:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078393.329206:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4099 00000100:00080000:2.0:1769078393.329216:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078393.329221:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078393.329250:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913633664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078393.329274:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078393.329277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078393.329278:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078393.329281:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4099 00000100:00080000:2.0:1769078393.329283:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078393.329285:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078393.329292:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078393.329299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913633792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078393.329307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078393.329310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078393.329311:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078393.329315:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4099 00000100:00080000:2.0:1769078393.329317:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078393.329320:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078393.329325:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078393.329334:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913633920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078393.329338:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078393.329344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078393.329348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078393.329350:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078393.329354:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4099 00000100:00080000:2.0:1769078393.329357:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078393.329360:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078393.329365:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078393.329375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913634048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078393.329394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078393.329398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078393.329400:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078393.329404:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4099 00000100:00080000:2.0:1769078393.329407:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078393.329411:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078393.329416:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078393.329431:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913633664:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078393.329431:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913634176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078393.329439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078393.329442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078393.329443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078393.329445:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4099 00000100:00080000:2.0:1769078393.329447:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078393.329449:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078393.329452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078393.329460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913634304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078393.329465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078393.329469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078393.329472:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078393.329476:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4099 00000100:00080000:2.0:1769078393.329478:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078393.329480:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078393.329483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078393.329489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913634432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078393.329651:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078393.329752:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078393/real 1769078393] req@ffff935002a62a00 x1855008913633664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078393.329785:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913633664/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078393.329822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078393.329828:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078393.329832:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913633664:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078393.329842:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633792:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078393.329866:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078393.329897:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078393/real 1769078393] req@ffff935002a60000 x1855008913633792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078393.329937:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913633792/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078393.329947:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078393.329951:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078393.329977:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633792:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078393.329984:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633920:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078393.330019:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078393.330042:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078393/real 1769078393] req@ffff935002a61500 x1855008913633920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078393.330085:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913633920/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078393.330109:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078393.330113:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078393.330116:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913633920:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078393.330123:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634048:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078393.330143:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078393.330191:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078393/real 1769078393] req@ffff935002a63100 x1855008913634048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078393.330218:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913634048/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078393.330240:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078393.330244:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078393.330268:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078393.330328:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913634176:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078393.330351:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078393.330377:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078393/real 1769078393] req@ffff935002a61c00 x1855008913634176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078393.330390:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913634176/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078393.330398:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078393.330401:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078393.330404:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913634176:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078393.330410:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913634304:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078393.330430:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078393.330441:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078393/real 1769078393] req@ffff935002a61f80 x1855008913634304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078393.330453:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913634304/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078393.330467:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078393.330470:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078393.330474:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913634304:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078393.330480:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913634432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078393.330498:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078393.330508:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078393/real 1769078393] req@ffff935002a60a80 x1855008913634432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078393.330519:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913634432/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078393.330526:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078393.330529:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078393.330532:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913634432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078393.521166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078394.033176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078394.033176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078394.033190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078394.033548:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078394.355781:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078394.545176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078395.057154:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078395.057164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078395.057998:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078395.058511:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078395.377160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078395.569172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078396.081151:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078396.081240:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078396.081275:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078396.081287:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078396.401181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078396.593843:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078397.105191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078397.105305:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078397.105475:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078397.105873:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078397.427872:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078397.617245:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078398.129159:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078398.129214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078398.129281:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078398.129343:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:0.0:1769078398.449234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078398.449247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078398.449253:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769078398.449253:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078398.449265:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4104 00000100:00080000:0.0:1769078398.449278:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078398.449285:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078398.449329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913634560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078398.449380:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913634560:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078398.449443:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078398.449473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078398.449477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078398.449479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078398.449483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4104 00000100:00080000:0.0:1769078398.449487:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078398.449491:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078398.449502:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078398.449515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913634688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078398.449569:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078398/real 1769078398] req@ffff9350103fad80 x1855008913634560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078398.449583:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913634560/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078398.449618:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078398.449622:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078398.449625:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913634560:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078398.449647:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634688:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078398.449674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078398.449695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078398/real 1769078398] req@ffff9350103fb480 x1855008913634688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078398.449719:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913634688/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078398.449728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078398.449731:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078398.449733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634688:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078398.449782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078398.449786:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078398.449788:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078398.449793:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4104 00000100:00080000:0.0:1769078398.449797:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078398.449801:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078398.449807:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078398.449816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913634816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078398.449823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078398.449825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078398.449827:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078398.449829:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4104 00000100:00080000:0.0:1769078398.449831:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078398.449833:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078398.449836:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078398.449841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913634944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078398.449846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078398.449848:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078398.449849:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078398.449851:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4104 00000100:00080000:0.0:1769078398.449853:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078398.449855:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078398.449857:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078398.449861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913635072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078398.449866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078398.449867:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078398.449868:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078398.449870:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4104 00000100:00080000:0.0:1769078398.449872:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078398.449873:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078398.449876:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078398.449883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913635200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078398.449887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078398.449889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078398.449890:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078398.449892:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4104 00000100:00080000:0.0:1769078398.449893:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078398.449896:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078398.449898:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078398.449905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913635328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078398.449966:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634816:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078398.449985:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078398.449993:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078398/real 1769078398] req@ffff9350103fb800 x1855008913634816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078398.450017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913634816/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078398.450024:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078398.450028:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078398.450031:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634816:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078398.450037:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078398.450076:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078398.450087:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078398/real 1769078398] req@ffff9350103f8700 x1855008913634944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078398.450099:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913634944/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078398.450114:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078398.450116:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078398.450118:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913634944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078398.450123:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635072:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078398.450163:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078398.450174:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078398/real 1769078398] req@ffff9350103faa00 x1855008913635072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078398.450186:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913635072/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078398.450195:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078398.450198:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078398.450201:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635072:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078398.450207:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635200:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078398.450225:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078398.450233:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078398/real 1769078398] req@ffff9350103fb480 x1855008913635200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078398.450240:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913635200/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078398.450245:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078398.450247:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078398.450250:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635200:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078398.450255:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078398.450265:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078398.450271:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078398/real 1769078398] req@ffff9350103fad80 x1855008913635328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078398.450277:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913635328/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078398.450282:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078398.450284:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078398.450286:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078398.641165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078399.153197:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078399.153197:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078399.153248:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078399.153648:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078399.473477:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078399.665208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078400.177190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078400.177329:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078400.177339:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078400.177489:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078400.497898:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078400.689274:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078401.201212:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078401.201212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078401.201283:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078401.201286:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078401.521460:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078401.713160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078402.225187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078402.225272:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078402.225386:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078402.225397:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078402.545407:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078402.737155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078403.249196:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078403.249344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078403.252304:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078403.252324:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:0.0:1769078403.570810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078403.570823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078403.570827:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078403.570837:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4109 00000100:00080000:0.0:1769078403.570849:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078403.570856:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078403.570859:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769078403.570895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913635456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078403.570922:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078403.570925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078403.570927:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078403.570932:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4109 00000100:00080000:0.0:1769078403.570935:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078403.570939:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078403.570948:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078403.570960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913635584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078403.570969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078403.570972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078403.570974:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078403.570978:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4109 00000100:00080000:0.0:1769078403.570981:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078403.570985:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078403.570989:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078403.570999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913635712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078403.571008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078403.571012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078403.571026:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078403.571030:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4109 00000100:00080000:0.0:1769078403.571033:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078403.571037:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078403.571042:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078403.571070:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913635840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078403.571079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078403.571082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078403.571084:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078403.571088:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4109 00000100:00080000:0.0:1769078403.571091:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078403.571094:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078403.571099:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078403.571108:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913635968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078403.571116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078403.571120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078403.571122:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078403.571125:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4109 00000100:00080000:0.0:1769078403.571129:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078403.571132:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078403.571137:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078403.571146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913636096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078403.571155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078403.571158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078403.571160:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078403.571163:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4109 00000100:00080000:0.0:1769078403.571165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078403.571169:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078403.571173:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078403.571183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913636224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078403.572384:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913635456:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078403.572543:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078403.572596:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078403/real 1769078403] req@ffff9350103f8380 x1855008913635456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078403.572618:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913635456/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078403.572629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078403.572634:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078403.572638:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913635456:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078403.572646:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913635584:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078403.572671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078403.572685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078403/real 1769078403] req@ffff9350103fb100 x1855008913635584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078403.572699:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913635584/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078403.572708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078403.572711:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078403.572715:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913635584:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078403.572722:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913635712:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078403.572742:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078403.572755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078403/real 1769078403] req@ffff9350103f9500 x1855008913635712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078403.572767:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913635712/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078403.572775:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078403.572778:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078403.572782:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913635712:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078403.572788:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913635840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078403.572808:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078403.572818:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078403/real 1769078403] req@ffff9350103fbb80 x1855008913635840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078403.572828:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913635840/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078403.572836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078403.572839:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078403.572842:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913635840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078403.572848:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635968:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078403.572867:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078403.572877:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078403/real 1769078403] req@ffff9350103f9c00 x1855008913635968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078403.572889:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913635968/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078403.572898:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078403.572901:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078403.572904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913635968:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078403.572910:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636096:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078403.572928:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078403.572939:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078403/real 1769078403] req@ffff9350103f9180 x1855008913636096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078403.572950:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913636096/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078403.572958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078403.572961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078403.572965:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636096:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078403.572971:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078403.572988:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078403.572998:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078403/real 1769078403] req@ffff9350103f9f80 x1855008913636224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078403.573010:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913636224/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078403.573018:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078403.573021:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078403.573024:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078403.761206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078404.273169:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078404.273186:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078404.273213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078404.273234:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078404.594638:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078404.785248:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078405.297208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078405.297213:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078405.297362:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078405.297377:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078405.617168:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078405.809184:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078406.321200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078406.321255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078406.321306:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078406.321654:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078406.641172:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078406.833401:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078407.345190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078407.345209:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078407.345316:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078407.345586:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078407.665160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078407.857928:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078408.369225:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078408.369255:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078408.369265:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078408.371549:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078408.691564:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078408.691667:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078408.691673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078408.691677:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078408.691688:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4114 00000100:00080000:3.0:1769078408.691700:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078408.691706:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078408.691752:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913636352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078408.691787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078408.691792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078408.691794:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078408.691798:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4114 00000100:00080000:3.0:1769078408.691802:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078408.691806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078408.691814:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078408.691825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913636480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078408.691834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078408.691837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078408.691838:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078408.691842:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4114 00000100:00080000:3.0:1769078408.691845:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078408.691848:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078408.691852:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078408.691859:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913636608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078408.691872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078408.691875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078408.691877:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078408.691880:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4114 00000100:00080000:3.0:1769078408.691883:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078408.691895:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078408.691899:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078408.691908:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913636736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078408.691915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078408.691918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078408.691920:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078408.691923:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4114 00000100:00080000:3.0:1769078408.691926:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078408.691930:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078408.691933:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078408.691942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913636864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078408.691951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078408.691954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078408.691956:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078408.691959:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4114 00000100:00080000:3.0:1769078408.691963:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078408.691966:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078408.691972:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078408.691973:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913636352:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078408.691983:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913636992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078408.691992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078408.691996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078408.691998:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078408.692001:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4114 00000100:00080000:3.0:1769078408.692004:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078408.692008:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078408.692011:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078408.692020:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913637120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078408.692044:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078408.692106:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078408/real 1769078408] req@ffff935002a7df80 x1855008913636352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078408.692124:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913636352/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078408.692133:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078408.692138:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078408.692142:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913636352:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078408.692149:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913636480:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078408.692172:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078408.692184:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078408/real 1769078408] req@ffff935002a7f800 x1855008913636480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078408.692197:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913636480/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078408.692207:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078408.692210:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078408.692213:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913636480:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078408.692219:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913636608:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078408.692238:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078408.692250:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078408/real 1769078408] req@ffff935002a7c000 x1855008913636608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078408.692262:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913636608/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078408.692271:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078408.692275:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078408.692278:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913636608:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078408.692284:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913636736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078408.692302:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078408.692312:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078408/real 1769078408] req@ffff935002a7ca80 x1855008913636736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078408.692324:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913636736/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078408.692332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078408.692335:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078408.692338:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913636736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078408.692343:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636864:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078408.692361:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078408.692371:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078408/real 1769078408] req@ffff935002a7f100 x1855008913636864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078408.692383:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913636864/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078408.692391:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078408.692394:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078408.692397:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636864:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078408.692442:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636992:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078408.692462:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078408.692472:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078408/real 1769078408] req@ffff935002a7c700 x1855008913636992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078408.692492:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913636992/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078408.692501:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078408.692504:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078408.692507:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913636992:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078408.692512:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913637120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078408.692529:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078408.692539:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078408/real 1769078408] req@ffff935002a7fb80 x1855008913637120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078408.692550:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913637120/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078408.692558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078408.692561:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078408.692564:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913637120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078408.881198:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078409.393226:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078409.393226:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078409.393239:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078409.393244:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078409.713201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078409.905691:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078410.417201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078410.417218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078410.417260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078410.417530:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078410.737166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078410.929193:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078411.441218:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078411.441219:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078411.441220:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078411.441266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078411.761196:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078411.953215:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078412.465206:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078412.465239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078412.465452:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078412.465470:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078412.785171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078412.977158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078413.489189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078413.489229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078413.492414:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078413.492472:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078413.809209:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078413.809230:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078413.809238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078413.809243:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078413.809255:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4119 00000100:00080000:2.0:1769078413.809266:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078413.809274:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078413.809318:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913637248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078413.809353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078413.809357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078413.809358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078413.809362:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4119 00000100:00080000:2.0:1769078413.809365:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078413.809369:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078413.809379:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078413.809390:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913637248:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078413.809395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913637376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078413.809413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078413.809417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078413.809422:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078413.809427:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4119 00000100:00080000:2.0:1769078413.809431:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078413.809435:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078413.809441:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078413.809452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913637504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078413.809461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078413.809465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078413.809466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078413.809473:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4119 00000800:00000100:0.0:1769078413.809475:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078413.809478:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078413.809483:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078413.809488:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078413.809499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913637632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078413.809507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078413.809521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078413.809523:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078413.809529:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078413/real 1769078413] req@ffff935002a62300 x1855008913637248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078413.809529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4119 00000100:00080000:2.0:1769078413.809535:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078413.809542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078413.809548:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078413.809558:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913637248/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078413.809562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913637760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078413.809584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078413.809584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078413.809591:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078413.809594:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769078413.809594:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078413.809603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913637248:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078413.809603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4119 00000100:00080000:2.0:1769078413.809607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078413.809612:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078413.809616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078413.809625:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913637888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078413.809635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078413.809636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913637376:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769078413.809641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078413.809645:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078413.809650:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4119 00000100:00080000:2.0:1769078413.809654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078413.809658:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078413.809663:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078413.809669:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078413.809676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913638016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078413.809685:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078413/real 1769078413] req@ffff935002a61880 x1855008913637376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078413.809714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913637376/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078413.809723:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078413.809727:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078413.809730:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913637376:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078413.809737:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913637504:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078413.809756:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078413.809767:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078413/real 1769078413] req@ffff935002a61180 x1855008913637504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078413.809777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913637504/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078413.809785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078413.809788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078413.809791:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913637504:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078413.809796:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913637632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078413.809814:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078413.809825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078413/real 1769078413] req@ffff935002a62680 x1855008913637632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078413.809836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913637632/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078413.809844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078413.809847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078413.809850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913637632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078413.809856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913637760:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078413.809872:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078413.809882:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078413/real 1769078413] req@ffff935002a60a80 x1855008913637760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078413.809892:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913637760/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078413.809900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078413.809902:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078413.809905:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913637760:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078413.809921:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913637888:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078413.809937:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078413.809947:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078413/real 1769078413] req@ffff935002a61f80 x1855008913637888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078413.809958:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913637888/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078413.809965:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078413.809968:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078413.809971:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913637888:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078413.809977:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078413.809994:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078413.810004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078413/real 1769078413] req@ffff935002a61c00 x1855008913638016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078413.810021:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913638016/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078413.810028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078413.810031:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078413.810034:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078414.001185:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078414.513164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078414.513176:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078414.515222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078414.515232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078414.833256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078415.025453:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078415.537208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078415.537216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078415.537227:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078415.537258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078415.857180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078416.049211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078416.561156:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078416.561178:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078416.561179:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078416.561230:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078416.881211:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078417.073158:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078417.585177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078417.585182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078417.585204:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078417.585232:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078417.905207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078418.097194:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078418.609185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078418.609185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078418.609198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078418.609221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078418.929164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078418.929167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078418.929175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078418.929180:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078418.929189:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4124 00000100:00080000:0.0:1769078418.929199:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078418.929206:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078418.929272:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913638144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078418.929295:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078418.929297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078418.929299:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078418.929302:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4124 00000100:00080000:0.0:1769078418.929305:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078418.929308:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078418.929318:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078418.929324:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913638144:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078418.929330:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913638272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078418.929337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078418.929340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078418.929341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078418.929344:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4124 00000100:00080000:0.0:1769078418.929345:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078418.929347:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078418.929350:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078418.929357:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913638400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078418.929362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078418.929365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078418.929366:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078418.929368:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4124 00000100:00080000:0.0:1769078418.929370:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078418.929373:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078418.929375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078418.929383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913638528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078418.929390:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078418.929391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078418.929395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078418.929397:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078418.929399:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4124 00000100:00080000:0.0:1769078418.929402:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078418.929404:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078418.929407:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078418.929413:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913638656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078418.929419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078418.929421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078418.929422:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078418.929424:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4124 00000100:00080000:0.0:1769078418.929427:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078418.929430:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078418.929433:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078418.929435:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9350103f9f80 x1855008913638144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078418.929442:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913638784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078418.929449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078418.929451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078418.929452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078418.929455:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913638144/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078418.929458:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4124 00000100:00080000:0.0:1769078418.929463:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078418.929468:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078418.929471:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769078418.929472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078418.929480:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078418.929483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913638912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078418.929485:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913638144:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078418.929514:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913638272:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078418.929531:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078418.929538:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9350103f9180 x1855008913638272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.929546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913638272/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.929552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.929554:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078418.929556:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913638272:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078418.929559:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913638400:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.929569:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078418.929574:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9350103f9c00 x1855008913638400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.929580:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913638400/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.929585:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.929586:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078418.929588:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913638400:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078418.929591:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913638528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.929600:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078418.929605:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9350103fbb80 x1855008913638528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.929611:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913638528/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.929615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.929617:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078418.929619:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913638528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078418.929623:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638656:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078418.929631:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078418.929636:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9350103f9500 x1855008913638656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.929642:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913638656/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.929647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.929648:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078418.929650:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638656:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078418.929653:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638784:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.929660:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078418.929665:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9350103fb100 x1855008913638784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.929671:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913638784/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.929675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.929677:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078418.929678:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638784:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078418.929681:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.929689:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078418.929695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9350103f8380 x1855008913638912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.929702:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913638912/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.929706:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.929708:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078418.929709:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913638912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078419.121115:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078419.633158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078419.633161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078419.633167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078419.633353:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078419.953207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078420.145186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078420.657171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078420.657177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078420.657552:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078420.658191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078420.977155:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078421.169189:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078421.681243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078421.681267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078421.681354:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078421.681553:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078422.001239:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078422.193231:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078422.705205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078422.705205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078422.705248:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078422.705380:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078423.025157:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078423.217188:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078423.729228:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078423.729277:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078423.729379:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078423.729490:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078424.049138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078424.049560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078424.049567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078424.049571:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078424.049580:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4129 00000100:00080000:2.0:1769078424.049591:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078424.049597:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078424.049635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913639040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078424.049663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078424.049667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078424.049669:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078424.049673:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4129 00000100:00080000:2.0:1769078424.049676:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078424.049680:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078424.049689:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078424.049702:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913639168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078424.049713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078424.049718:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913639040:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078424.049719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078424.049722:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078424.049730:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4129 00000100:00080000:2.0:1769078424.049745:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078424.049748:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078424.049753:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078424.049762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913639296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078424.049769:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078424.049771:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078424.049772:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078424.049774:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4129 00000100:00080000:2.0:1769078424.049779:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078424.049781:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078424.049785:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1769078424.049790:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078424.049797:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913639424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078424.049808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078424.049811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078424.049813:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078424.049817:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4129 00000100:00080000:2.0:1769078424.049822:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078424.049826:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078424.049831:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078424/real 1769078424] req@ffff935002a63100 x1855008913639040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769078424.049831:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078424.049845:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913639552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078424.049853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078424.049854:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913639040/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078424.049856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078424.049860:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078424.049865:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4129 00000100:00080000:2.0:1769078424.049871:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078424.049874:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769078424.049875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769078424.049877:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078424.049885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913639680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078424.049893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078424.049897:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769078424.049898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078424.049902:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078424.049906:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913639040:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078424.049909:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4129 00000100:00080000:2.0:1769078424.049916:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078424.049932:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078424.049935:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913639168:192.168.204.121@tcp:38:kworker.0 00000008:00100000:2.0:1769078424.049939:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078424.049953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913639808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078424.049964:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078424.049981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078424/real 1769078424] req@ffff935002a61500 x1855008913639168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078424.049993:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913639168/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078424.050002:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078424.050005:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078424.050009:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913639168:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078424.050014:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913639296:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078424.050030:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078424.050072:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078424/real 1769078424] req@ffff935002a60000 x1855008913639296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078424.050086:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913639296/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078424.050094:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078424.050097:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078424.050101:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913639296:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078424.050107:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913639424:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078424.050124:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078424.050134:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078424/real 1769078424] req@ffff935002a62a00 x1855008913639424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078424.050146:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913639424/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078424.050154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078424.050169:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078424.050172:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913639424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078424.050179:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913639552:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078424.050199:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078424.050210:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078424/real 1769078424] req@ffff935002a61c00 x1855008913639552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078424.050221:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913639552/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078424.050229:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078424.050232:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078424.050235:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913639552:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078424.050241:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913639680:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078424.050259:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078424.050269:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078424/real 1769078424] req@ffff935002a61f80 x1855008913639680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078424.050280:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913639680/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078424.050287:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078424.050290:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078424.050306:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913639680:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078424.050323:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913639808:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078424.050340:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078424.050349:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078424/real 1769078424] req@ffff935002a60a80 x1855008913639808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078424.050360:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913639808/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078424.050368:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078424.050372:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078424.050375:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913639808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078424.241199:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078424.754445:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078424.754562:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078424.755065:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078424.755157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078425.073254:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078425.265177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078425.777185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078425.777185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078425.777300:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078425.777417:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078426.097169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078426.289563:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078426.801143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078426.801150:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078426.801197:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078426.801226:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078427.121159:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078427.313161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078427.825177:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078427.825177:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078427.825193:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078427.825222:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078428.145173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078428.337201:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078428.849182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078428.849226:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078428.849237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078428.850242:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078429.169206:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078429.169293:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078429.169298:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078429.169302:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078429.169309:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4134 00000100:00080000:1.0:1769078429.169320:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078429.169325:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078429.169361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913639936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078429.169402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078429.169406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078429.169408:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078429.169413:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4134 00000100:00080000:1.0:1769078429.169416:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078429.169420:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078429.169429:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078429.169441:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913640064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078429.169450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078429.169453:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078429.169455:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078429.169459:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4134 00000100:00080000:1.0:1769078429.169462:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078429.169465:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078429.169469:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078429.169477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913640192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078429.169494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078429.169497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078429.169499:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078429.169504:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4134 00000100:00080000:1.0:1769078429.169507:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078429.169510:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078429.169515:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078429.169524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913640320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078429.169532:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078429.169536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078429.169537:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078429.169541:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4134 00000100:00080000:1.0:1769078429.169544:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078429.169547:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078429.169551:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078429.169560:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913640448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078429.169568:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078429.169571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078429.169572:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078429.169575:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4134 00000100:00080000:1.0:1769078429.169578:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078429.169582:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078429.169586:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078429.169594:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913640576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078429.169602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078429.169605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078429.169606:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078429.169609:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4134 00000100:00080000:1.0:1769078429.169612:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078429.169615:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078429.169619:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078429.169627:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913640704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078429.169708:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913639936:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769078429.169780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078429.169818:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078429/real 1769078429] req@ffff9350100c9c00 x1855008913639936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078429.169833:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913639936/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078429.169842:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078429.169846:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078429.169850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913639936:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078429.169857:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640064:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078429.169875:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078429.169898:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078429/real 1769078429] req@ffff9350100c9500 x1855008913640064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078429.169910:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913640064/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078429.169918:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078429.169921:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078429.169924:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640064:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078429.169929:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640192:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078429.169946:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078429.169955:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078429/real 1769078429] req@ffff9350100caa00 x1855008913640192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078429.169965:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913640192/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078429.169972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078429.169976:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078429.169979:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640192:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078429.169984:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078429.169999:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078429.170008:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078429/real 1769078429] req@ffff9350100c8380 x1855008913640320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078429.170018:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913640320/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078429.170026:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078429.170029:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078429.170032:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078429.170036:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913640448:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078429.170088:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078429.170099:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078429/real 1769078429] req@ffff9350100ca680 x1855008913640448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078429.170110:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913640448/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078429.170118:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078429.170121:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078429.170124:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913640448:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078429.170130:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913640576:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078429.170145:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078429.170153:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078429/real 1769078429] req@ffff9350100c8e00 x1855008913640576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078429.170164:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913640576/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078429.170176:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078429.170180:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078429.170183:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913640576:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078429.170188:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913640704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078429.170203:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078429.170213:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078429/real 1769078429] req@ffff9350100ca300 x1855008913640704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078429.170224:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913640704/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078429.170232:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078429.170235:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078429.170238:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913640704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078429.361494:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078429.873187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078429.873199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078429.873210:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078429.873249:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078430.193207:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078430.385171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078430.897177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078430.897178:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078430.897179:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078430.897212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078431.217161:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078431.409154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078431.921166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078431.921166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078431.921178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078431.921209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078432.241181:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078432.433125:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078432.945180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078432.945210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078432.949143:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078432.949238:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078433.266164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078433.457157:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078433.969189:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078433.969189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078433.970142:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078433.970190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078434.289228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078434.289240:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078434.289240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078434.289246:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078434.289259:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4139 00000100:00080000:2.0:1769078434.289270:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078434.289277:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078434.289316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913640832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078434.289345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078434.289349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078434.289350:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078434.289354:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4139 00000100:00080000:2.0:1769078434.289358:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078434.289363:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078434.289373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078434.289388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913640960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078434.289395:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913640832:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078434.289402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078434.289407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078434.289408:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078434.289414:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4139 00000100:00080000:2.0:1769078434.289418:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078434.289421:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078434.289432:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078434.289445:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913641088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078434.289454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078434.289458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078434.289460:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078434.289464:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4139 00000100:00080000:2.0:1769078434.289467:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078434.289472:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:2.0:1769078434.289475:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078434.289481:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078434.289493:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913641216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078434.289502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078434.289506:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078434.289507:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078434.289511:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4139 00000100:00080000:2.0:1769078434.289515:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078434.289520:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078434.289525:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078434.289526:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078434/real 1769078434] req@ffff935002a62680 x1855008913640832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078434.289541:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913641344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078434.289548:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913640832/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078434.289562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078434.289565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078434.289569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078434.289571:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078434.289574:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769078434.289578:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4139 00000100:00100000:0.0:1769078434.289583:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913640832:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078434.289584:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078434.289592:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078434.289598:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078434.289610:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913641472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078434.289614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640960:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769078434.289626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078434.289630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078434.289632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078434.289636:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4139 00000100:00080000:2.0:1769078434.289640:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078434.289644:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078434.289649:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078434.289650:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078434.289665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913641600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078434.289669:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078434/real 1769078434] req@ffff935002a61180 x1855008913640960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078434.289693:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913640960/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078434.289701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078434.289705:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078434.289708:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913640960:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078434.289715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641088:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078434.289732:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078434.289741:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078434/real 1769078434] req@ffff935002a61880 x1855008913641088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078434.289752:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913641088/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078434.289764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078434.289767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078434.289770:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641088:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078434.289791:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641216:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078434.289810:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078434.289820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078434/real 1769078434] req@ffff935002a62300 x1855008913641216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078434.289831:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913641216/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078434.289838:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078434.289842:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078434.289845:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078434.289850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913641344:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078434.289865:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078434.289875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078434/real 1769078434] req@ffff935002a60a80 x1855008913641344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078434.289885:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913641344/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078434.289893:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078434.289895:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078434.289899:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913641344:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078434.289914:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913641472:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078434.289928:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078434.289937:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078434/real 1769078434] req@ffff935002a61f80 x1855008913641472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078434.289948:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913641472/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078434.289955:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078434.289958:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078434.289961:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913641472:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078434.289966:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913641600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078434.289980:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078434.290005:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078434/real 1769078434] req@ffff935002a61c00 x1855008913641600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078434.290017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913641600/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078434.290029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078434.290032:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078434.290035:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913641600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078434.481210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078434.993158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078434.993158:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078434.993258:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078434.996110:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078435.313192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078435.505801:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078436.017155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078436.018597:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078436.019704:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078436.019907:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078436.337165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078436.529151:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078437.041161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078437.041260:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078437.041296:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078437.041301:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078437.361403:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078437.553154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078438.065191:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078438.065204:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078438.065300:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078438.065443:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078438.385145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078438.577128:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078439.089118:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078439.089220:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078439.092779:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078439.092843:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078439.409180:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078439.409220:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078439.409228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078439.409233:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078439.409242:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4145 00000100:00080000:2.0:1769078439.409253:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078439.409259:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078439.409297:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913641728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078439.409323:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078439.409325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078439.409326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078439.409368:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913641728:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078439.409415:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4145 00000100:00080000:2.0:1769078439.409420:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078439.409424:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078439.409433:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078439.409446:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913641856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078439.409457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078439.409460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078439.409462:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078439.409466:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4145 00000100:00080000:2.0:1769078439.409469:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078439.409473:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078439.409477:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078439.409488:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913641984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078439.409497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078439.409500:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078439.409502:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078439.409505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4145 00000100:00080000:2.0:1769078439.409509:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078439.409512:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078439.409516:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078439.409526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913642112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078439.409534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078439.409537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078439.409538:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078439.409542:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4145 00000100:00080000:2.0:1769078439.409545:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078439.409549:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:0.0:1769078439.409601:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:2.0:1769078439.409638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078439.409653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078439/real 1769078439] req@ffff935002a62a00 x1855008913641728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078439.409657:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913642240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078439.409688:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913641728/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078439.409688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078439.409696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078439.409699:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078439.409709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078439.409709:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4145 00000100:00080000:0.0:1769078439.409717:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:2.0:1769078439.409717:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078439.409727:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913641728:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078439.409727:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078439.409735:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078439.409749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913642368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078439.409759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078439.409761:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078439.409762:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078439.409763:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641856:192.168.204.121@tcp:38:kworker.0 00000100:00080000:2.0:1769078439.409766:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4145 00000100:00080000:2.0:1769078439.409772:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078439.409776:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078439.409780:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078439.409790:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913642496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078439.409797:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078439.409815:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078439/real 1769078439] req@ffff935002a60000 x1855008913641856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078439.409827:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913641856/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078439.409836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078439.409840:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078439.409844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641856:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078439.409850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641984:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078439.409866:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078439.409875:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078439/real 1769078439] req@ffff935002a61500 x1855008913641984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078439.409897:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913641984/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078439.409933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078439.409937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078439.409940:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913641984:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078439.409947:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913642112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078439.409965:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078439.409976:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078439/real 1769078439] req@ffff935002a63100 x1855008913642112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078439.409987:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913642112/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078439.409995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078439.409999:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078439.410002:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913642112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078439.410008:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913642240:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078439.410023:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078439.410031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078439/real 1769078439] req@ffff935002a61c00 x1855008913642240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078439.410084:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913642240/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078439.410094:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078439.410097:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078439.410101:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913642240:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078439.410117:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913642368:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078439.410136:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078439.410146:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078439/real 1769078439] req@ffff935002a61f80 x1855008913642368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078439.410158:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913642368/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078439.410166:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078439.410169:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078439.410172:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913642368:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078439.410178:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913642496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078439.410196:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078439.410206:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078439/real 1769078439] req@ffff935002a60a80 x1855008913642496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078439.410217:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913642496/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078439.410231:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078439.410234:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078439.410238:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913642496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078439.601216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078440.113156:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078440.113306:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078440.113399:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078440.113585:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078440.433169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078440.625272:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078441.137191:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078441.137318:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078441.137379:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078441.137591:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078441.457149:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078441.649197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078442.161201:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078442.162104:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078442.162249:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078442.162752:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078442.481195:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078442.673623:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078443.185149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078443.185149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078443.185180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078443.185630:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078443.505296:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078443.697162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078444.209349:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078444.209848:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078444.209934:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078444.209977:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078444.529165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078444.529741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078444.529748:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078444.529751:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078444.529759:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4150 00000100:00080000:2.0:1769078444.529769:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078444.529775:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078444.529809:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913642624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078444.529847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078444.529850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078444.529852:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078444.529856:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4150 00000100:00080000:2.0:1769078444.529858:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078444.529862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078444.529870:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078444.529881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913642752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078444.529890:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078444.529892:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078444.529894:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078444.529897:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4150 00000100:00080000:2.0:1769078444.529899:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078444.529901:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078444.529904:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078444.529912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913642880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078444.529919:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078444.529922:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078444.529923:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078444.529926:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4150 00000100:00080000:2.0:1769078444.529929:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078444.529932:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078444.529935:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078444.529943:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913643008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078444.529950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078444.529953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078444.529954:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078444.529957:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4150 00000100:00080000:2.0:1769078444.529960:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078444.529962:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078444.529965:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078444.529972:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913643136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078444.529979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078444.529982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078444.529983:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078444.529985:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4150 00000100:00080000:2.0:1769078444.529987:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078444.529990:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078444.529994:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078444.530001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913643264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078444.530008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078444.530011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078444.530012:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078444.530015:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4150 00000100:00080000:2.0:1769078444.530017:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078444.530020:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078444.530023:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078444.530029:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913643392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078444.530864:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913642624:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769078444.530926:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078444.530961:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078444/real 1769078444] req@ffff935002a62300 x1855008913642624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078444.530974:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913642624/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078444.530981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078444.530984:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078444.530986:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913642624:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078444.530991:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913642752:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078444.531005:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078444.531012:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078444/real 1769078444] req@ffff935002a61880 x1855008913642752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078444.531024:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913642752/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078444.531029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078444.531031:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078444.531032:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913642752:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078444.531036:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913642880:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078444.531045:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078444.531069:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078444/real 1769078444] req@ffff935002a61180 x1855008913642880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078444.531077:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913642880/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078444.531081:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078444.531083:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078444.531084:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913642880:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078444.531088:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078444.531099:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078444.531105:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078444/real 1769078444] req@ffff935002a62680 x1855008913643008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078444.531111:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913643008/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078444.531120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078444.531122:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078444.531124:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078444.531127:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913643136:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078444.531135:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078444.531140:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078444/real 1769078444] req@ffff935002a60a80 x1855008913643136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078444.531146:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913643136/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078444.531151:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078444.531153:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078444.531154:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913643136:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078444.531157:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913643264:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078444.531166:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078444.531171:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078444/real 1769078444] req@ffff935002a61f80 x1855008913643264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078444.531177:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913643264/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078444.531182:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078444.531183:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078444.531186:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913643264:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078444.531189:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913643392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078444.531197:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078444.531202:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078444/real 1769078444] req@ffff935002a61c00 x1855008913643392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078444.531208:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913643392/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078444.531212:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078444.531214:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078444.531215:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913643392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078444.722648:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078445.233140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078445.233144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078445.233186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078445.233206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078445.553123:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078445.745163:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078446.257158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078446.257159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078446.257188:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078446.257192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078446.577248:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078446.769204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078447.281185:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078447.281215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078447.281533:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078447.282513:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078447.601447:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078447.793425:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078448.305207:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078448.305213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078448.305654:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078448.305812:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078448.625158:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078448.817182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078449.329201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078449.329700:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078449.329796:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078449.329832:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00080000:0.0:1769078449.649941:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078449.649952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078449.649956:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078449.649966:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4155 00000100:00080000:0.0:1769078449.649979:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078449.649985:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078449.650022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913643520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078449.650090:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913643520:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078449.650237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078449.650241:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078449.650244:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078449.650248:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4155 00000100:00080000:0.0:1769078449.650252:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078449.650256:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078449.650264:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078449.650277:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913643648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078449.650414:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:2.0:1769078449.653995:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643648:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078449.654196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078449.654202:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078449.654205:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078449.654211:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4155 00000100:00080000:0.0:1769078449.654216:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078449.654222:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078449.654233:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078449.654260:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913643776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078449.654275:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078449.654277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078449.654279:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078449.654283:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4155 00000100:00080000:0.0:1769078449.654286:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078449.654289:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078449.654293:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078449.654304:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913643904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078449.654311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078449.654314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078449.654316:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078449.654319:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4155 00000100:00080000:0.0:1769078449.654322:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078449.654325:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078449.654330:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078449.654338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913644032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078449.654347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078449.654350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078449.654351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078449.654354:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4155 00000100:00080000:0.0:1769078449.654357:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078449.654360:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078449.654363:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078449.654374:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913644160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078449.654390:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078449.654393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078449.654395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078449.654398:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4155 00000100:00080000:0.0:1769078449.654401:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078449.654404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078449.654409:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078449.654418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913644288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078449.654533:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078449.654626:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078449.654649:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078449.654669:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078449.654687:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078449.841357:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078450.353121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078450.353174:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078450.353229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078450.353236:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078450.675837:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078450.865168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078451.377190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078451.377287:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078451.377360:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078451.378218:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078451.697446:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078451.889131:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078452.401168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078452.401191:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078452.401343:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078452.401748:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078452.721535:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:0.0:1769078452.722395:0:2350:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:0.0:1769078452.722438:0:2350:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769078452.722457:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769078452.722500:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769078452.722510:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769078452.722521:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769078452.722536:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078452.722545:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078449/real 1769078452] req@ffff9350103faa00 x1855008913643776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1769078452.722552:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:0.0:1769078452.722564:0:2350:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078452.722579:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913643776/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078452.722592:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078452.722598:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078452.722603:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643776:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078452.722610:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078449/real 1769078452] req@ffff9350103f8700 x1855008913643904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078452.722622:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913643904/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078452.722629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078452.722632:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078452.722635:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078452.722642:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078449/real 1769078452] req@ffff9350103fb800 x1855008913644032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078452.722660:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913644032/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078452.722668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078452.722671:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078452.722688:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644032:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078452.722695:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078449/real 1769078452] req@ffff9350103f8380 x1855008913644160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078452.722707:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913644160/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078452.722715:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078452.722718:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078452.722721:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644160:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078452.722728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078449/real 1769078452] req@ffff9350103fb100 x1855008913644288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078452.722739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913644288/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078452.722747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078452.722750:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078452.722753:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078452.722759:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078449/real 1769078452] req@ffff9350103fb480 x1855008913643648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078452.722770:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913643648/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078452.722778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078452.722781:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078452.722785:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913643648:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078452.722791:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078449/real 1769078452] req@ffff9350103fad80 x1855008913643520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078452.722803:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913643520/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078452.722810:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078452.722813:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078452.722816:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913643520:192.168.204.121@tcp:250:kworker.0 00000100:00100000:1.0:1769078452.913197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078453.425172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078453.425215:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078453.425325:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078453.425344:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078453.745399:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078453.937162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078454.449925:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078454.450020:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078454.450098:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078454.450840:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078454.769155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078454.769163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078454.769167:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078454.769176:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4160 00000100:00080000:3.0:1769078454.769186:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078454.769192:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078454.769203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078454.769234:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913644416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078454.769259:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078454.769263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078454.769265:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078454.769268:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4160 00000100:00080000:3.0:1769078454.769271:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078454.769274:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078454.769284:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078454.769297:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913644544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078454.769304:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913644416:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078454.769309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078454.769314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078454.769316:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078454.769320:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4160 00000100:00080000:3.0:1769078454.769325:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078454.769329:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078454.769335:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078454.769348:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913644672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078454.769356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078454.769359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078454.769361:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078454.769364:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4160 00000100:00080000:3.0:1769078454.769381:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078454.769385:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078454.769390:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078454.769392:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078454.769407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913644800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078454.769417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078454.769420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078454.769422:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078454.769425:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4160 00000100:00080000:3.0:1769078454.769429:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078454.769433:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078454.769439:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078454.769445:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078454/real 1769078454] req@ffff935002a7c380 x1855008913644416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078454.769455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913644928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078454.769466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078454.769470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078454.769472:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078454.769476:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913644416/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078454.769479:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4160 00000100:00080000:3.0:1769078454.769486:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078454.769494:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078454.769495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769078454.769500:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078454.769504:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078454.769511:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913644416:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078454.769516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913645056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078454.769526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078454.769529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078454.769531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078454.769536:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4160 00000100:00100000:2.0:1769078454.769539:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913644544:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078454.769541:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078454.769550:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078454.769556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078454.769569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913645184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078454.769576:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078454.769600:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078454/real 1769078454] req@ffff935002a7dc00 x1855008913644544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078454.769613:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913644544/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078454.769622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078454.769625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078454.769628:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913644544:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078454.769634:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913644672:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078454.769681:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078454.769693:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078454/real 1769078454] req@ffff935002a7d180 x1855008913644672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078454.769705:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913644672/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078454.769712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078454.769715:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078454.769718:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913644672:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078454.769724:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913644800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078454.769744:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078454.769755:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078454/real 1769078454] req@ffff935002a7e300 x1855008913644800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078454.769765:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913644800/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078454.769773:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078454.769776:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078454.769779:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913644800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078454.769785:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644928:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078454.769803:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078454.769812:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078454/real 1769078454] req@ffff935002a7fb80 x1855008913644928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078454.769823:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913644928/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078454.769830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078454.769833:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078454.769836:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913644928:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078454.769841:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645056:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078454.769858:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078454.769868:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078454/real 1769078454] req@ffff935002a7c700 x1855008913645056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078454.769899:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913645056/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078454.769907:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078454.769910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078454.769913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645056:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078454.769937:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078454.769963:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078454.769974:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078454/real 1769078454] req@ffff935002a7f100 x1855008913645184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078454.769984:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913645184/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078454.769990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078454.769993:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078454.769995:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078454.961182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078455.473190:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078455.473190:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078455.473208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078455.473233:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078455.793272:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078455.985236:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078456.497187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078456.497187:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078456.497187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078456.497237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078456.817779:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078457.009854:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078457.521162:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078457.521162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078457.521206:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078457.521266:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078457.841414:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078458.033170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078458.545183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078458.545190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078458.545207:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078458.545223:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078458.865616:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078459.059090:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078459.569126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078459.569127:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078459.569181:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078459.570639:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:1.0:1769078459.890671:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078459.890683:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078459.890687:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078459.890696:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4165 00000100:00080000:1.0:1769078459.890708:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078459.890715:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078459.890762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913645312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078459.890791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078459.890795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078459.890797:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078459.890802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4165 00000100:00080000:1.0:1769078459.890805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078459.890809:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078459.890819:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078459.890832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913645440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078459.890842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078459.890845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078459.890846:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078459.890850:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4165 00000100:00080000:1.0:1769078459.890854:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078459.890857:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078459.890863:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078459.890873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913645568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078459.890904:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078459.890907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078459.890909:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078459.890913:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4165 00000100:00080000:1.0:1769078459.890916:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078459.890919:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078459.890924:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078459.890953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913645696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078459.890962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078459.890965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078459.890967:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078459.890970:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4165 00000100:00080000:1.0:1769078459.890974:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078459.890977:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078459.890982:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078459.890992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913645824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078459.891005:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078459.891008:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078459.891010:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078459.891013:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4165 00000100:00080000:1.0:1769078459.891016:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078459.891019:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078459.891024:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078459.891033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913645952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078459.891041:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078459.891045:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078459.891046:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078459.891154:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913645312:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078459.891192:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000800:00000100:0.0:1769078459.891221:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078459.891260:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff9350100cb800 x1855008913645312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078459.891275:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913645312/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078459.891289:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078459.891294:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078459.891298:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913645312:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078459.891306:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913645440:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078459.891329:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078459.891341:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff9350100ca300 x1855008913645440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078459.891352:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913645440/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078459.891361:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078459.891364:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078459.891367:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913645440:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078459.891372:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913645568:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078459.891392:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078459.891403:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff9350100c8e00 x1855008913645568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078459.891416:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913645568/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078459.891429:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078459.891432:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078459.891435:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913645568:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078459.891441:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913645696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078459.891458:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078459.891469:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff9350100ca680 x1855008913645696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078459.891480:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913645696/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078459.891490:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078459.891494:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078459.891497:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913645696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078459.891503:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645824:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078459.891521:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078459.891532:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff9350100c8380 x1855008913645824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078459.891543:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913645824/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078459.891551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078459.891565:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078459.891569:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645824:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078459.891574:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645952:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078459.891593:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078459.891604:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff9350100caa00 x1855008913645952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078459.891615:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913645952/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078459.891622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078459.891625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078459.891629:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913645952:192.168.204.121@tcp:8:kworker.0 00000100:00080000:1.0:1769078459.891693:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4165 00000100:00080000:1.0:1769078459.891698:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078459.891702:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078459.891707:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078459.891719:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913646080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078459.892001:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078459.892023:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078459.892035:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff9350100c9500 x1855008913646080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078459.892047:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913646080/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078459.892074:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078459.892077:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078459.892080:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078460.081162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078460.593167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078460.593183:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078460.593287:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078460.593351:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078460.913280:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078461.105190:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078461.617125:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078461.617146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078461.617170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078461.617227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078461.937197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078462.129182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078462.641168:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078462.641196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078462.641477:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078462.642369:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078462.961201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078463.153146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078463.665190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078463.665203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078463.665415:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078463.666446:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078463.985169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078464.177167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078464.689158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078464.689187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078464.689256:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078464.690560:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:3.0:1769078465.010696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078465.010702:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078465.010713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078465.010718:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078465.010732:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4170 00000100:00080000:3.0:1769078465.010746:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078465.010756:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078465.010797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913646208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078465.010834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078465.010838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078465.010840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078465.010844:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4170 00000100:00080000:3.0:1769078465.010848:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078465.010852:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078465.010863:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078465.010875:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913646336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078465.010885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078465.010888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078465.010890:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078465.010894:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4170 00000100:00080000:3.0:1769078465.010897:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078465.010901:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078465.010906:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078465.010916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913646464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078465.010924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078465.010928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078465.010930:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078465.010933:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4170 00000100:00080000:3.0:1769078465.010936:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078465.010940:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078465.010944:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078465.010954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913646592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078465.010968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078465.010972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078465.010973:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078465.010977:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4170 00000100:00080000:3.0:1769078465.010981:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078465.010984:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078465.010989:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078465.011001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913646720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078465.011009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078465.011012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078465.011014:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078465.011018:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4170 00000100:00080000:3.0:1769078465.011021:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078465.011024:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078465.011028:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078465.011038:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913646848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078465.011045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078465.011142:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913646208:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769078465.011208:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078465.011249:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078465/real 1769078465] req@ffff935002a7ca80 x1855008913646208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078465.011264:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913646208/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078465.011272:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078465.011276:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078465.011279:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913646208:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078465.011286:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913646336:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078465.011304:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078465.011315:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078465/real 1769078465] req@ffff935002a7c000 x1855008913646336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078465.011326:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913646336/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078465.011333:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078465.011335:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078465.011339:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913646336:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078465.011343:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913646464:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078465.011359:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078465.011368:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078465/real 1769078465] req@ffff935002a7f800 x1855008913646464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078465.011377:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913646464/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078465.011384:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078465.011386:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078465.011389:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913646464:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078465.011393:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913646592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078465.011406:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078465.011413:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078465/real 1769078465] req@ffff935002a7df80 x1855008913646592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078465.011422:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913646592/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078465.011430:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078465.011433:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078465.011436:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913646592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078465.011440:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646720:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078465.011454:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078465.011463:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078465/real 1769078465] req@ffff935002a7f100 x1855008913646720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078465.011486:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913646720/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078465.011494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078465.011497:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078465.011500:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646720:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078465.011506:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646848:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078465.011525:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078465.011535:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078465/real 1769078465] req@ffff935002a7c700 x1855008913646848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078465.011546:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913646848/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078465.011553:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078465.011556:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078465.011559:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646848:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769078465.011628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078465.011630:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078465.011633:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4170 00000100:00080000:3.0:1769078465.011636:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078465.011639:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078465.011643:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078465.011652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913646976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078465.011728:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078465.011751:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078465.011762:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078465/real 1769078465] req@ffff935002a7fb80 x1855008913646976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078465.011778:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913646976/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078465.011786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078465.011789:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078465.011793:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913646976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078465.201195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078465.713170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078465.713197:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078465.713304:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078465.714251:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078466.033189:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078466.226146:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078466.737187:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078466.737187:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078466.737189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078466.738645:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078467.057178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078467.249346:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078467.761158:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078467.761201:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078467.761314:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078467.761375:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078468.081154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078468.273175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078468.785166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078468.785166:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078468.785166:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078468.785213:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078469.105176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078469.297175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078469.809166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078469.809269:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078469.809347:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078469.814830:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078470.129276:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078470.129289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078470.129513:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078470.129525:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4175 00000100:00080000:0.0:1769078470.129537:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078470.129545:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078470.129582:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913647104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078470.129610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078470.129614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078470.129616:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078470.129620:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4175 00000100:00080000:0.0:1769078470.129623:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078470.129627:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078470.129635:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078470.129647:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913647232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078470.129657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078470.129661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078470.129663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078470.129666:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4175 00000100:00080000:0.0:1769078470.129669:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078470.129672:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078470.129676:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078470.129684:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913647360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078470.129693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078470.129695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078470.129697:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078470.129701:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4175 00000100:00080000:0.0:1769078470.129704:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078470.129707:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078470.129710:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078470.129717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913647488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078470.129725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078470.129728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078470.129730:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078470.129733:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4175 00000100:00080000:0.0:1769078470.129735:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078470.129738:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078470.129743:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078470.129751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913647616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078470.129761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078470.129774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078470.129776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078470.129780:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4175 00000100:00080000:0.0:1769078470.129783:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078470.129786:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078470.129791:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078470.129800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913647744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078470.129809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078470.129813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078470.129815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078470.129818:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4175 00000100:00080000:0.0:1769078470.129820:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078470.129823:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078470.129827:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078470.129835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913647872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078470.130617:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913647104:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078470.130682:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078470.130720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078470/real 1769078470] req@ffff9350103f9500 x1855008913647104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078470.130739:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913647104/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078470.130749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078470.130754:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078470.130758:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913647104:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078470.130766:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913647232:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078470.130786:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078470.130798:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078470/real 1769078470] req@ffff9350103fbb80 x1855008913647232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078470.130810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913647232/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078470.130820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078470.130823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078470.130827:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913647232:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078470.130834:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913647360:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078470.130851:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078470.130861:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078470/real 1769078470] req@ffff9350103f9c00 x1855008913647360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078470.130874:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913647360/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078470.130882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078470.130885:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078470.130889:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913647360:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078470.130895:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913647488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078470.130912:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078470.130921:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078470/real 1769078470] req@ffff9350103f9180 x1855008913647488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078470.130932:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913647488/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078470.130940:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078470.130944:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078470.130947:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913647488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078470.130954:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913647616:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078470.130971:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078470.130981:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078470/real 1769078470] req@ffff9350103f9f80 x1855008913647616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078470.130992:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913647616/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078470.131000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078470.131004:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078470.131007:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913647616:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078470.131013:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913647744:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078470.131029:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078470.131038:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078470/real 1769078470] req@ffff9350103fad80 x1855008913647744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078470.131069:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913647744/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078470.131078:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078470.131081:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078470.131085:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913647744:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078470.131092:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913647872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078470.131111:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078470.131121:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078470/real 1769078470] req@ffff9350103fb480 x1855008913647872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078470.131133:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913647872/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078470.131147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078470.131150:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078470.131154:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913647872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078470.131261:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078470.321237:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078470.833155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078470.833160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078470.834139:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078470.834673:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078471.153201:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078471.345170:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078471.857200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078471.857200:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078471.857453:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078471.859474:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078472.177236:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078472.369220:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078472.881134:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078472.881146:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078472.881166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078472.881199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078473.201128:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078473.393182:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078473.905183:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078473.905183:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078473.905183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078473.905217:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078474.225146:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078474.417183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078474.929161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078474.929161:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078474.929175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078474.929194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078475.249141:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078475.249155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078475.249161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078475.249167:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078475.249180:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4180 00000100:00080000:0.0:1769078475.249194:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078475.249203:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078475.249251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913648000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078475.249290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078475.249307:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078475.249309:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078475.249322:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4180 00000100:00080000:0.0:1769078475.249327:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078475.249332:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078475.249345:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078475.249351:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913648000:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078475.249364:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913648128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078475.249393:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078475.249397:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078475.249399:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078475.249405:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4180 00000100:00080000:0.0:1769078475.249409:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078475.249414:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078475.249422:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1769078475.249429:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078475.249439:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913648256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078475.249452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078475.249456:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078475.249458:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078475.249462:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4180 00000100:00080000:0.0:1769078475.249467:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078475.249472:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078475.249477:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078475.249485:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078475/real 1769078475] req@ffff9350103fb100 x1855008913648000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078475.249493:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913648384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078475.249506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078475.249511:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913648000/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078475.249512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078475.249517:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078475.249526:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4180 00000100:00080000:0.0:1769078475.249533:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078475.249534:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078475.249541:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:2.0:1769078475.249544:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000008:00100000:0.0:1769078475.249549:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078475.249552:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913648000:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078475.249565:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913648512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078475.249576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078475.249579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078475.249581:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078475.249587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913648128:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078475.249588:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4180 00000100:00080000:0.0:1769078475.249595:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078475.249601:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078475.249615:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078475.249633:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913648640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078475.249634:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078475.249649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078475.249653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078475.249658:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078475.249658:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078475/real 1769078475] req@ffff9350103f8380 x1855008913648128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078475.249666:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4180 00000100:00080000:0.0:1769078475.249673:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078475.249679:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078475.249684:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913648128/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769078475.249694:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078475.249697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078475.249702:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078475.249704:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913648128:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078475.249713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913648768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078475.249714:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913648256:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078475.249747:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078475.249761:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078475/real 1769078475] req@ffff9350103fb800 x1855008913648256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078475.249775:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913648256/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078475.249784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078475.249788:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078475.249792:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913648256:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078475.249807:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913648384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078475.249828:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078475.249836:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078475/real 1769078475] req@ffff9350103f8700 x1855008913648384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078475.249848:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913648384/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078475.249857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078475.249860:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078475.249863:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913648384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078475.249870:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913648512:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078475.249890:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078475.249902:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078475/real 1769078475] req@ffff9350103faa00 x1855008913648512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078475.249913:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913648512/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078475.249919:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078475.249922:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078475.249924:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913648512:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078475.249938:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913648640:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078475.249953:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078475.249959:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078475/real 1769078475] req@ffff9350103fb100 x1855008913648640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078475.249967:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913648640/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078475.249972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078475.249974:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078475.249976:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913648640:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078475.249989:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913648768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078475.250003:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078475.250010:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078475/real 1769078475] req@ffff9350103fb480 x1855008913648768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078475.250017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913648768/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078475.250030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078475.250033:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078475.250037:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913648768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078475.441206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078475.953199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078475.953202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078475.953243:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078475.953260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078476.273222:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078476.465171:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078476.977123:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078476.977165:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078476.977191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078476.977253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078477.297214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078477.489209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078478.001148:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078478.001148:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078478.001172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078478.001194:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078478.321218:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078478.513212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078479.025186:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078479.025205:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078479.025362:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078479.025410:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078479.345393:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078479.537209:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078480.049159:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078480.049203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078480.049229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078480.049242:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00080000:0.0:1769078480.369197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078480.369200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078480.369233:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078480.369238:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078480.369252:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4186 00000100:00080000:0.0:1769078480.369263:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078480.369271:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078480.369320:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913648896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078480.369417:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913648896:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078480.369491:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078480.369526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078480.369530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078480.369531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078480.369536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4186 00000100:00080000:0.0:1769078480.369540:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078480.369544:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078480.369555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078480.369568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913649024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078480.369629:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078480/real 1769078480] req@ffff9350103fad80 x1855008913648896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078480.369648:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913648896/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078480.369657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078480.369662:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078480.369666:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913648896:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078480.369690:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649024:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078480.369711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078480.369722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078480/real 1769078480] req@ffff9350103f9f80 x1855008913649024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078480.369734:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913649024/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078480.369749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078480.369752:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078480.369755:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649024:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078480.369801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078480.369805:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078480.369808:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078480.369812:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4186 00000100:00080000:0.0:1769078480.369816:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078480.369820:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078480.369825:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078480.369837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913649152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078480.369853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078480.369856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078480.369858:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078480.369861:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4186 00000100:00080000:0.0:1769078480.369864:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078480.369867:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078480.369871:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078480.369878:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913649280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078480.369887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078480.369890:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078480.369892:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078480.369895:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4186 00000100:00080000:0.0:1769078480.369897:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078480.369900:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078480.369904:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078480.369911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913649408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078480.369918:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078480.369921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078480.369923:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078480.369926:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4186 00000100:00080000:0.0:1769078480.369929:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078480.369932:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078480.369936:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078480.369943:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913649536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078480.369951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078480.369954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078480.369967:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078480.369970:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4186 00000100:00080000:0.0:1769078480.369973:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078480.369976:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078480.369980:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078480.369988:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913649664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078480.370107:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649152:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078480.370130:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078480.370142:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078480/real 1769078480] req@ffff9350103f9180 x1855008913649152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078480.370153:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913649152/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078480.370160:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078480.370164:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078480.370167:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649152:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078480.370172:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078480.370189:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078480.370199:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078480/real 1769078480] req@ffff9350103f9c00 x1855008913649280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078480.370222:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913649280/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078480.370231:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078480.370234:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078480.370238:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078480.370243:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913649408:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078480.370262:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078480.370272:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078480/real 1769078480] req@ffff9350103fbb80 x1855008913649408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078480.370283:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913649408/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078480.370291:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078480.370294:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078480.370297:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913649408:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078480.370302:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913649536:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078480.370318:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078480.370327:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078480/real 1769078480] req@ffff9350103f9500 x1855008913649536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078480.370338:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913649536/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078480.370345:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078480.370348:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078480.370351:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913649536:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078480.370356:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913649664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078480.370371:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078480.370380:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078480/real 1769078480] req@ffff9350103f9f80 x1855008913649664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078480.370391:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913649664/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078480.370399:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078480.370402:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078480.370405:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913649664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078480.562988:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078481.073238:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078481.073238:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078481.073239:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078481.074293:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078481.393394:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078481.585252:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078482.097175:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078482.097175:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078482.097996:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078482.098334:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078482.417267:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078482.609211:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078483.121192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078483.121200:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078483.121211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078483.121396:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078483.441928:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078483.633179:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078484.145246:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078484.145725:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078484.146382:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078484.146788:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078484.465276:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078484.657206:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078485.169177:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078485.169374:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078485.169414:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078485.169492:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078485.489172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078485.489186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1769078485.489187:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078485.489192:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078485.489323:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4191 00000100:00080000:3.0:1769078485.489336:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078485.489343:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078485.489382:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913649792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078485.489410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078485.489414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078485.489425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078485.489430:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4191 00000100:00080000:3.0:1769078485.489435:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078485.489439:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078485.489449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078485.489455:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913649792:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078485.489469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913649920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078485.489491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078485.489495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078485.489496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078485.489501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4191 00000100:00080000:3.0:1769078485.489519:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078485.489523:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078485.489528:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:3.0:1769078485.489529:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078485.489547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913650048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078485.489558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078485.489562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078485.489565:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078485.489570:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4191 00000100:00100000:2.0:1769078485.489574:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078485/real 1769078485] req@ffff935002a7e300 x1855008913649792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078485.489577:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078485.489592:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078485.489598:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078485.489602:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913649792/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078485.489614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078485.489619:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913650176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078485.489623:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078485.489632:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913649792:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078485.489655:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078485.489659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078485.489661:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078485.489668:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4191 00000100:00080000:3.0:1769078485.489672:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078485.489673:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649920:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078485.489687:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078485.489694:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078485.489698:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078485.489710:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078485/real 1769078485] req@ffff935002a7d180 x1855008913649920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078485.489718:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913649920/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078485.489727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078485.489734:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913650304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078485.489735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078485.489740:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913649920:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078485.489754:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650048:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769078485.489777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769078485.489780:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078485.489787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769078485.489790:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078485/real 1769078485] req@ffff935002a7dc00 x1855008913650048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078485.489792:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078485.489812:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4191 00000100:00100000:2.0:1769078485.489813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913650048/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078485.489824:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078485.489826:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078485.489836:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078485.489840:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078485.489844:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650048:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078485.489853:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650176:192.168.204.121@tcp:8:kworker.0 00000008:00100000:3.0:1769078485.489854:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078485.489871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078485.489877:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078485/real 1769078485] req@ffff935002a7c380 x1855008913650176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078485.489886:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913650176/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078485.489892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078485.489896:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078485.489898:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650176:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078485.489909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913650304:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078485.489918:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078485.489924:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078485/real 1769078485] req@ffff935002a7e300 x1855008913650304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078485.489930:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913650304/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078485.489935:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078485.489937:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078485.489939:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913650304:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078485.490534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913650432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078485.490560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078485.490564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078485.490567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078485.490572:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4191 00000100:00080000:3.0:1769078485.490578:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078485.490582:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078485.490590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078485.490608:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913650560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078485.490614:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913650432:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078485.490656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078485.490667:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078485/real 1769078485] req@ffff935002a7fb80 x1855008913650432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078485.490676:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913650432/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078485.490681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078485.490683:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078485.490686:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913650432:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078485.490699:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913650560:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078485.490711:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078485.490720:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078485/real 1769078485] req@ffff935002a7c700 x1855008913650560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078485.490730:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913650560/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078485.490738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078485.490741:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078485.490744:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913650560:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078485.681365:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078486.193404:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078486.193415:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078486.193598:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078486.194796:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078486.513349:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078486.705221:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078487.217211:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078487.217211:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078487.217214:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078487.218107:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078487.537233:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078487.729239:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078488.241205:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078488.241295:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078488.241300:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078488.241306:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078488.561443:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078488.753501:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078489.265156:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078489.265217:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078489.265233:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078489.265299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078489.585186:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078489.777196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078490.289231:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078490.289364:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078490.289371:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078490.291306:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078490.609151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078490.609154:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078490.609161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078490.609166:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078490.609177:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4196 00000100:00080000:3.0:1769078490.609189:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078490.609197:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078490.609242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913650688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078490.609279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078490.609283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078490.609285:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078490.609289:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4196 00000100:00080000:3.0:1769078490.609293:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078490.609296:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078490.609307:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078490.609318:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913650816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078490.609329:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078490.609332:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078490.609334:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078490.609337:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4196 00000100:00080000:3.0:1769078490.609340:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078490.609344:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078490.609348:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078490.609356:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913650944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078490.609365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078490.609368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078490.609370:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078490.609373:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4196 00000100:00080000:3.0:1769078490.609376:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078490.609380:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078490.609384:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078490.609391:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913651072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078490.609400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078490.609403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078490.609405:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078490.609408:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4196 00000100:00080000:3.0:1769078490.609410:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078490.609414:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078490.609418:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078490.609426:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913651200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078490.609434:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078490.609437:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078490.609439:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078490.609443:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4196 00000100:00080000:3.0:1769078490.609445:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078490.609448:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078490.609451:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078490.609462:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913651328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078490.609475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078490.609478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078490.609480:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078490.609483:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4196 00000100:00080000:3.0:1769078490.609486:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078490.609489:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078490.609492:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078490.609499:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913651456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078490.609585:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913650688:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078490.609656:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078490.609698:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078490/real 1769078490] req@ffff935002a7f100 x1855008913650688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078490.609714:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913650688/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078490.609724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078490.609728:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078490.609732:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913650688:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078490.609739:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650816:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078490.609758:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078490.609769:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078490/real 1769078490] req@ffff935002a7df80 x1855008913650816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078490.609780:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913650816/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078490.609789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078490.609792:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078490.609795:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650816:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078490.609801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650944:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078490.609817:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078490.609825:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078490/real 1769078490] req@ffff935002a7f800 x1855008913650944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078490.609836:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913650944/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078490.609844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078490.609847:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078490.609850:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913650944:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078490.609856:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651072:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078490.609870:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078490.609878:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078490/real 1769078490] req@ffff935002a7c000 x1855008913651072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078490.609889:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913651072/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078490.609897:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078490.609900:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078490.609904:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651072:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078490.609909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913651200:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078490.609923:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078490.609931:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078490/real 1769078490] req@ffff935002a7ca80 x1855008913651200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078490.609943:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913651200/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078490.609951:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078490.609954:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078490.609957:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913651200:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078490.609962:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913651328:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078490.609975:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078490.609984:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078490/real 1769078490] req@ffff935002a7c700 x1855008913651328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078490.609995:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913651328/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078490.610009:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078490.610012:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078490.610015:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913651328:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078490.610020:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913651456:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078490.610033:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078490.610098:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078490/real 1769078490] req@ffff935002a7fb80 x1855008913651456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078490.610109:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913651456/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078490.610117:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078490.610120:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078490.610123:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913651456:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078490.801210:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078491.313231:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078491.313439:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078491.314277:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078491.314438:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078491.633832:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078491.825196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078492.337150:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078492.337150:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078492.343189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078492.343249:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078492.657182:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078492.849183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078493.361160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078493.361160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078493.361261:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078493.363238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078493.681256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078493.873536:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078494.385208:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078494.385221:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078494.385253:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078494.385447:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078494.705214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078494.897172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078495.409202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078495.409242:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078495.409370:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078495.409448:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078495.729199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078495.729409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078495.729416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078495.729420:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078495.729430:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4201 00000100:00080000:1.0:1769078495.729443:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078495.729452:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078495.729491:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913651584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078495.729524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078495.729529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078495.729531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078495.729535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4201 00000100:00080000:1.0:1769078495.729539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078495.729543:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078495.729554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078495.729568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913651712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078495.729577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078495.729580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078495.729583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078495.729586:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4201 00000100:00080000:1.0:1769078495.729589:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078495.729592:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078495.729597:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078495.729607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913651840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078495.729615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078495.729618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078495.729620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078495.729623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4201 00000100:00080000:1.0:1769078495.729626:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078495.729630:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078495.729634:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078495.729642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913651968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078495.729649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078495.729652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078495.729654:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078495.729657:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4201 00000100:00080000:1.0:1769078495.729660:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078495.729663:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078495.729667:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078495.729675:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913652096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078495.729683:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078495.729686:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078495.729688:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078495.729692:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4201 00000100:00080000:1.0:1769078495.729694:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078495.729697:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078495.729701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078495.729710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913652224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078495.729729:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078495.729733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078495.729735:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078495.729738:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4201 00000100:00080000:1.0:1769078495.729742:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078495.729745:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078495.729749:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078495.729758:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913652352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078495.729835:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913651584:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078495.729993:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078495.730032:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078495/real 1769078495] req@ffff9350100c9c00 x1855008913651584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078495.730076:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913651584/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078495.730094:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078495.730098:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078495.730102:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913651584:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078495.730185:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651712:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078495.730210:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078495.730223:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078495/real 1769078495] req@ffff9350100cb100 x1855008913651712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078495.730234:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913651712/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078495.730242:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078495.730245:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078495.730248:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651712:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078495.730254:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651840:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078495.730270:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078495.730279:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078495/real 1769078495] req@ffff9350100c9180 x1855008913651840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078495.730290:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913651840/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078495.730303:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078495.730306:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078495.730308:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651840:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078495.730314:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651968:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078495.730330:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078495.730340:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078495/real 1769078495] req@ffff9350100c8a80 x1855008913651968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078495.730350:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913651968/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078495.730358:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078495.730360:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078495.730363:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913651968:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078495.730368:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652096:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078495.730384:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078495.730392:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078495/real 1769078495] req@ffff9350100c8000 x1855008913652096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078495.730402:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913652096/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078495.730409:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078495.730412:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078495.730415:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652096:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078495.730420:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652224:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078495.730434:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078495.730443:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078495/real 1769078495] req@ffff9350100c9880 x1855008913652224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078495.730454:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913652224/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078495.730460:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078495.730463:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078495.730466:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652224:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078495.730471:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652352:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078495.730485:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078495.730494:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078495/real 1769078495] req@ffff9350100cad80 x1855008913652352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078495.730504:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913652352/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078495.730511:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078495.730514:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078495.730517:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652352:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078495.921155:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078496.433165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078496.433165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078496.433286:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078496.433321:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078496.753178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078496.949075:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078497.457181:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078497.457696:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078497.457830:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078497.457887:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078497.777179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078497.970764:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078498.481158:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078498.481187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078498.481294:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078498.481324:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078498.801164:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078498.993183:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078499.505128:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078499.505156:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078499.505241:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078499.506415:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078499.825174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078500.017164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078500.529182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078500.529227:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078500.529356:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078500.531229:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078500.849245:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078500.849317:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078500.849324:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078500.849328:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078500.849338:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4206 00000100:00080000:3.0:1769078500.849349:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078500.849356:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078500.849395:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913652480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078500.849430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078500.849433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078500.849435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078500.849439:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4206 00000100:00080000:3.0:1769078500.849442:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078500.849446:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078500.849455:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078500.849467:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913652608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078500.849476:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078500.849479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078500.849481:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078500.849485:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4206 00000100:00080000:3.0:1769078500.849488:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078500.849492:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078500.849497:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078500.849507:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913652736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078500.849515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078500.849518:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078500.849520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078500.849523:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4206 00000100:00080000:3.0:1769078500.849526:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078500.849529:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078500.849533:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078500.849542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913652864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078500.849550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078500.849552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078500.849554:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078500.849557:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4206 00000100:00080000:3.0:1769078500.849560:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078500.849563:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078500.849568:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078500.849579:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913652992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078500.849587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078500.849589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078500.849591:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078500.849594:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4206 00000100:00080000:3.0:1769078500.849597:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078500.849600:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078500.849604:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078500.849614:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913653120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078500.849622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078500.849625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078500.849627:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078500.849630:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4206 00000100:00080000:3.0:1769078500.849633:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078500.849637:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078500.849641:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078500.849651:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913653248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078500.849726:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913652480:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078500.849784:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078500.849817:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff935002a7e300 x1855008913652480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078500.849831:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913652480/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078500.849840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078500.849844:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078500.849847:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913652480:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078500.849852:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913652608:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078500.849882:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078500.849893:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff935002a7c380 x1855008913652608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078500.849903:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913652608/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078500.849912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078500.849915:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078500.849918:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913652608:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078500.849924:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913652736:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078500.849938:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078500.849946:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff935002a7dc00 x1855008913652736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078500.849956:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913652736/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078500.849968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078500.849971:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078500.849973:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913652736:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078500.849978:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913652864:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078500.849991:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078500.849998:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff935002a7d180 x1855008913652864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078500.850007:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913652864/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078500.850015:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078500.850018:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078500.850020:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913652864:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078500.850025:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652992:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078500.850038:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078500.850046:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff935002a7fb80 x1855008913652992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078500.850084:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913652992/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078500.850091:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078500.850094:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078500.850098:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913652992:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078500.850103:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913653120:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078500.850119:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078500.850127:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff935002a7c700 x1855008913653120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078500.850138:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913653120/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078500.850144:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078500.850147:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078500.850150:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913653120:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078500.850155:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913653248:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078500.850168:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078500.850175:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff935002a7ca80 x1855008913653248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078500.850186:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913653248/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078500.850192:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078500.850195:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078500.850197:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913653248:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078501.041234:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078501.553210:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078501.553799:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078501.554098:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078501.554171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078501.873837:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078502.065167:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078502.577170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078502.577276:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078502.577299:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078502.578816:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078502.897166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078503.089205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078503.601168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078503.601191:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078503.601209:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078503.601298:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078503.921583:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078504.113593:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078504.625241:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:3.0:1769078504.625774:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078504.625809:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078504.625900:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078504.945369:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078505.137175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078505.649167:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078505.649289:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078505.649361:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078505.649835:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078505.969166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078505.969170:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078505.969179:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078505.969183:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.969193:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4211 00000100:00080000:3.0:1769078505.969203:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.969210:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078505.969249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913653376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078505.969281:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.969286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078505.969287:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.969292:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4211 00000100:00080000:3.0:1769078505.969295:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.969298:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078505.969307:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078505.969318:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913653504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078505.969325:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913653376:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078505.969333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.969337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078505.969339:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.969343:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4211 00000100:00080000:3.0:1769078505.969346:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.969349:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078505.969354:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078505.969365:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913653632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078505.969373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.969377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078505.969378:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.969382:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4211 00000100:00080000:3.0:1769078505.969385:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078505.969388:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078505.969393:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078505.969398:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078505.969409:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913653760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078505.969419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.969423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078505.969425:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078505.969427:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff935002a7c000 x1855008913653376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078505.969431:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4211 00000100:00080000:3.0:1769078505.969437:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.969443:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078505.969447:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078505.969448:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913653376/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078505.969459:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078505.969460:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913653888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078505.969464:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078505.969471:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913653376:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078505.969481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.969485:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078505.969487:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.969490:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4211 00000100:00080000:3.0:1769078505.969494:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078505.969497:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913653504:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078505.969501:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078505.969508:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078505.969519:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078505.969521:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913654016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078505.969531:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff935002a7f800 x1855008913653504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078505.969534:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.969542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078505.969546:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078505.969550:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913653504/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078505.969552:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4211 00000100:00080000:3.0:1769078505.969556:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.969560:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078505.969567:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078505.969571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078505.969576:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078505.969579:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913653504:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078505.969583:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913654144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078505.969587:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913653632:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078505.969607:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078505.969617:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff935002a7df80 x1855008913653632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078505.969624:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913653632/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078505.969629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078505.969631:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078505.969633:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913653632:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078505.969636:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913653760:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078505.969644:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078505.969649:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff935002a7f100 x1855008913653760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078505.969655:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913653760/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078505.969660:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078505.969662:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078505.969663:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913653760:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078505.969666:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913653888:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078505.969674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078505.969678:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff935002a7ca80 x1855008913653888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078505.969684:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913653888/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078505.969689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078505.969691:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078505.969693:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913653888:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078505.969706:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654016:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078505.969714:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078505.969719:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff935002a7c700 x1855008913654016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078505.969725:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913654016/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078505.969730:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078505.969731:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078505.969733:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654016:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078505.969736:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654144:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078505.969743:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078505.969748:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff935002a7fb80 x1855008913654144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078505.969754:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913654144/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078505.969758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078505.969759:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078505.969761:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654144:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078506.161819:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078506.673289:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078506.673396:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078506.673404:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078506.676136:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078506.993171:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078507.185180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078507.697121:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078507.697125:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078507.697172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078507.697212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078508.017174:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078508.209168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078508.721143:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078508.721143:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078508.721235:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078508.721247:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078509.041176:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078509.233144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078509.745109:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078509.745114:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078509.745114:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078509.745195:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078510.065136:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078510.257160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078510.769106:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078510.769115:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078510.769146:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078510.769164:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078511.089198:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078511.089202:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078511.089214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078511.089218:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078511.089226:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4216 00000100:00080000:1.0:1769078511.089236:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078511.089243:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078511.089286:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9f80 x1855008913654272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078511.089318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078511.089322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078511.089324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078511.089327:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4216 00000100:00080000:1.0:1769078511.089331:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078511.089335:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078511.089344:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078511.089358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913654400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078511.089367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078511.089370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078511.089372:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078511.089375:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4216 00000100:00080000:1.0:1769078511.089378:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078511.089380:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078511.089384:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078511.089391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913654528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078511.089396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078511.089398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078511.089399:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078511.089401:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4216 00000100:00080000:1.0:1769078511.089402:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078511.089404:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078511.089406:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078511.089412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913654656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078511.089416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078511.089418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078511.089419:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078511.089422:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4216 00000100:00080000:1.0:1769078511.089424:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078511.089428:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078511.089432:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078511.089441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913654784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078511.089449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078511.089453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078511.089454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078511.089458:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4216 00000100:00080000:1.0:1769078511.089473:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078511.089475:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078511.089479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078511.089487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913654912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078511.089494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078511.089497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078511.089499:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078511.089501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4216 00000100:00080000:1.0:1769078511.089504:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078511.089507:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078511.089511:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078511.089521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913655040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078511.089604:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913654272:192.168.204.121@tcp:250:kworker.0 00000800:00000100:2.0:1769078511.089674:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078511.089727:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078511/real 1769078511] req@ffff9350100c9f80 x1855008913654272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078511.089743:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9f80 x1855008913654272/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078511.089753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078511.089756:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078511.089759:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913654272:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078511.089766:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913654400:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078511.089788:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078511.089799:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078511/real 1769078511] req@ffff9350100cad80 x1855008913654400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078511.089810:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913654400/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078511.089819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078511.089822:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078511.089825:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913654400:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078511.089831:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913654528:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078511.089850:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078511.089860:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078511/real 1769078511] req@ffff9350100c9880 x1855008913654528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078511.089872:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913654528/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078511.089885:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078511.089888:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078511.089890:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913654528:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078511.089894:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913654656:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078511.089907:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078511.089914:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078511/real 1769078511] req@ffff9350100c8000 x1855008913654656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078511.089921:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913654656/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078511.089926:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078511.089928:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078511.089929:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913654656:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078511.089932:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654784:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078511.089945:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078511.089950:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078511/real 1769078511] req@ffff9350100c8a80 x1855008913654784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078511.089957:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913654784/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078511.089962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078511.089964:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078511.089966:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654784:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078511.089968:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654912:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078511.089977:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078511.089982:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078511/real 1769078511] req@ffff9350100c9180 x1855008913654912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078511.089997:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913654912/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078511.090004:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078511.090007:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078511.090010:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913654912:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078511.090015:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655040:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078511.090032:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078511.090042:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078511/real 1769078511] req@ffff9350100cb100 x1855008913655040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078511.090079:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913655040/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078511.090092:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078511.090095:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078511.090098:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655040:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078511.281174:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078511.793144:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078511.793144:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078511.793146:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078511.793184:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078512.113197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078512.305175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078512.817180:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078512.817180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078512.817402:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:0.0:1769078512.817411:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078513.137120:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078513.331150:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078513.841212:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078513.841216:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078513.841462:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078513.841526:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078514.161175:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078514.353168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078514.865160:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078514.865160:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078514.865160:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078514.865196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078515.185228:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078515.377240:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078515.889172:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078515.889240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078515.889439:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078515.889591:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078516.209746:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078516.209750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078516.209759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078516.209769:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078516.209777:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4221 00000100:00080000:0.0:1769078516.209787:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078516.209793:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078516.209824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913655168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078516.209845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078516.209848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078516.209849:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078516.209852:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4221 00000100:00080000:0.0:1769078516.209854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078516.209856:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078516.209864:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078516.209871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913655296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078516.209878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078516.209881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078516.209882:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078516.209885:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4221 00000100:00080000:0.0:1769078516.209887:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078516.209889:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078516.209891:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078516.209897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913655424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078516.209905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078516.209908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769078516.209909:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913655168:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078516.209910:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078516.209917:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4221 00000100:00080000:0.0:1769078516.209920:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078516.209922:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078516.209933:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078516.209950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913655552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078516.209959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078516.209963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078516.209965:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078516.209969:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4221 00000100:00080000:0.0:1769078516.209973:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078516.209978:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078516.209983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078516.209993:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913655680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078516.209999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078516.210002:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078516.210003:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078516.210005:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4221 00000100:00080000:0.0:1769078516.210008:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078516.210011:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078516.210014:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078516.210021:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913655808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078516.210027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078516.210030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078516.210031:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078516.210033:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4221 00000100:00080000:0.0:1769078516.210035:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078516.210037:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078516.210039:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078516.210047:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913655936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078516.210087:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913655296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078516.210143:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913655424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078516.210181:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913655552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078516.210209:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078516.210239:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078516.210266:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078516.401164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078516.913183:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078516.913184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078516.913380:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:1.0:1769078516.913541:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078517.233284:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078517.425180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078517.937167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078517.937168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078517.937244:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078517.937254:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078518.257286:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078518.450818:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078518.961168:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078518.961196:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078518.961413:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078518.961633:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078519.281584:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000400:00000100:3.0:1769078519.281703:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.121:988 00000400:00000100:3.0:1769078519.281769:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.121@tcp at host 192.168.204.121 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078519.281796:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:2.0:1769078519.283679:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078516/real 1769078519] req@ffff9350103fad80 x1855008913655168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078541 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078519.283722:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913655168/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078541 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078519.283734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078519.283740:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078519.283744:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913655168:192.168.204.121@tcp:250:kworker.0 00000800:00000100:3.0:1769078519.286779:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078519.286812:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078519.286824:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078519.286833:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769078519.286841:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078516/real 1769078519] req@ffff9350103fb100 x1855008913655424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769078519.286843:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000800:00000100:3.0:1769078519.286857:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.121@tcp 00000100:00100000:0.0:1769078519.286883:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913655424/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078519.286897:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078519.286903:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078519.286908:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913655424:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078519.286916:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078516/real 1769078519] req@ffff9350103faa00 x1855008913655552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078519.286927:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913655552/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078519.286932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078519.286934:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078519.286937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913655552:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078519.286940:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078516/real 1769078519] req@ffff9350103f8700 x1855008913655680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078519.286950:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913655680/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078519.286958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078519.286961:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078519.286964:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655680:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078519.286971:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078516/real 1769078519] req@ffff9350103fb800 x1855008913655808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078519.286983:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913655808/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078519.286990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078519.286993:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078519.286996:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655808:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078519.287004:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078516/real 1769078519] req@ffff9350103f8380 x1855008913655936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078519.287016:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913655936/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078519.287023:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078519.287027:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078519.287030:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913655936:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078519.287037:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078516/real 1769078519] req@ffff9350103fb480 x1855008913655296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078519.287092:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913655296/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078519.287103:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078519.287159:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078519.287163:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913655296:192.168.204.121@tcp:38:kworker.0 00000100:00100000:1.0:1769078519.473970:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078519.985188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078519.985188:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078519.985292:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078519.990349:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078520.305266:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078520.499758:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078521.009441:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078521.009990:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078521.010203:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078521.010471:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078521.329217:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078521.329245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078521.329256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078521.329262:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078521.329273:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4227 00000100:00080000:0.0:1769078521.329286:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078521.329305:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078521.329349:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb480 x1855008913656064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078521.329393:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078521.329398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078521.329400:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078521.329405:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4227 00000100:00080000:0.0:1769078521.329409:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078521.329413:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078521.329425:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078521.329440:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8380 x1855008913656192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078521.329451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078521.329454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078521.329457:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078521.329462:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4227 00000100:00080000:0.0:1769078521.329468:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078521.329475:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913656064:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078521.329477:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078521.329487:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078521.329502:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb800 x1855008913656320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078521.329515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078521.329543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078521.329546:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078521.329551:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4227 00000100:00080000:0.0:1769078521.329556:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078521.329560:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078521.329565:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078521.329576:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f8700 x1855008913656448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078521.329585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078521.329588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078521.329590:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078521.329593:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4227 00000100:00080000:0.0:1769078521.329596:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078521.329600:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078521.329604:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078521.329614:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103faa00 x1855008913656576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078521.329623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078521.329627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078521.329628:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078521.329633:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4227 00000100:00080000:0.0:1769078521.329637:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078521.329645:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078521.329648:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078521.329658:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078521.329674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fb100 x1855008913656704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078521.329700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078521.329703:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078521/real 1769078521] req@ffff9350103fb480 x1855008913656064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078521.329706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078521.329710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078521.329725:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4227 00000100:00080000:0.0:1769078521.329732:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078521.329736:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078521.329743:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078521.329744:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb480 x1855008913656064/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078521.329763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769078521.329766:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913656832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078521.329774:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078521.329789:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913656064:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078521.329801:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913656192:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078521.329830:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078521.329844:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078521/real 1769078521] req@ffff9350103f8380 x1855008913656192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078521.329858:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8380 x1855008913656192/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078521.329867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078521.329871:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078521.329875:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913656192:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078521.329902:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913656320:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078521.329924:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078521.329935:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078521/real 1769078521] req@ffff9350103fb800 x1855008913656320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078521.329947:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb800 x1855008913656320/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078521.329954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078521.329958:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078521.329961:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913656320:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078521.329968:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913656448:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078521.329985:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078521.329996:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078521/real 1769078521] req@ffff9350103f8700 x1855008913656448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078521.330007:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f8700 x1855008913656448/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078521.330014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078521.330017:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078521.330020:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913656448:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078521.330026:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913656576:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078521.330043:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078521.330109:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078521/real 1769078521] req@ffff9350103faa00 x1855008913656576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078521.330122:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103faa00 x1855008913656576/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078521.330129:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078521.330132:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078521.330135:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913656576:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078521.330141:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913656704:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078521.330163:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078521.330173:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078521/real 1769078521] req@ffff9350103fb100 x1855008913656704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078521.330185:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fb100 x1855008913656704/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078521.330193:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078521.330197:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078521.330200:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913656704:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078521.330209:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913656832:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078521.330230:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078521.330252:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078521/real 1769078521] req@ffff9350103fad80 x1855008913656832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078521.330266:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913656832/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078521.330280:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078521.330285:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078521.330288:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913656832:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078521.521161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078522.033195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078522.033195:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078522.033269:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078522.033404:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078522.353420:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078522.545195:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078523.057218:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078523.057253:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078523.057266:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078523.057275:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078523.377214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078523.569447:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078524.081162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078524.081162:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078524.081286:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078524.083152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078524.401199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078524.593587:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078525.105168:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078525.105178:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078525.110415:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078525.137810:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078525.425880:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078525.617187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078526.129773:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078526.129848:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078526.130144:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078526.193319:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078526.449279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078526.449289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078526.449293:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078526.449301:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4232 00000100:00080000:3.0:1769078526.449311:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078526.449317:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078526.449348:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913656960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078526.449368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078526.449371:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078526.449373:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078526.449376:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4232 00000100:00080000:3.0:1769078526.449378:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078526.449381:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078526.449389:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078526.449398:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913657088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078526.449405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078526.449411:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913656960:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078526.449418:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078526.449420:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078526.449424:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4232 00000100:00080000:3.0:1769078526.449427:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078526.449429:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078526.449444:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000008:00100000:3.0:1769078526.449452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078526.449463:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913657216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078526.449473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078526.449476:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078526.449479:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078526.449484:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078526.449487:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4232 00000100:00080000:3.0:1769078526.449493:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078526.449519:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078526/real 1769078526] req@ffff935002a7d180 x1855008913656960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078526.449532:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913656960/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078526.449542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078526.449546:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078526.449550:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913656960:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078526.449572:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657088:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078526.449581:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078526.449587:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078526.449598:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913657344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078526.449610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769078526.449611:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078526.449617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078526.449620:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078526.449625:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4232 00000100:00100000:2.0:1769078526.449630:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078526/real 1769078526] req@ffff935002a7dc00 x1855008913657088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078526.449631:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078526.449640:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078526.449649:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078526.449655:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913657088/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078526.449665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913657472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078526.449667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078526.449676:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00080000:3.0:1769078526.449680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078526.449681:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657088:192.168.204.121@tcp:38:kworker.0 00000100:00080000:3.0:1769078526.449686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078526.449688:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078526.449692:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657216:192.168.204.121@tcp:8:kworker.0 00000100:00080000:3.0:1769078526.449695:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4232 00000100:00080000:3.0:1769078526.449699:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078526.449702:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078526.449706:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078526.449715:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078526.449718:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913657600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078526.449726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078526.449728:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078526/real 1769078526] req@ffff935002a7c380 x1855008913657216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078526.449729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078526.449732:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078526.449737:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4232 00000100:00080000:3.0:1769078526.449742:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078526.449746:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913657216/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078526.449747:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078526.449752:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078526.449761:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078526.449764:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913657728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078526.449768:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078526.449777:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657216:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078526.449795:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657344:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078526.449813:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078526.449824:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078526/real 1769078526] req@ffff935002a7e300 x1855008913657344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078526.449835:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913657344/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078526.449847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078526.449851:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078526.449854:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657344:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078526.449859:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913657472:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078526.449877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078526.449887:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078526/real 1769078526] req@ffff935002a7d180 x1855008913657472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078526.449896:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913657472/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078526.449904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078526.449906:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078526.449909:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913657472:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078526.449914:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913657600:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078526.449931:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078526.449939:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078526/real 1769078526] req@ffff935002a7fb80 x1855008913657600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078526.449949:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913657600/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078526.449969:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078526.449972:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078526.449976:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913657600:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078526.449981:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913657728:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078526.449997:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078526.450007:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078526/real 1769078526] req@ffff935002a7c700 x1855008913657728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078526.450017:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913657728/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078526.450024:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078526.450027:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078526.450029:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913657728:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078526.641585:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078527.153140:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078527.153140:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078527.154514:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078527.219586:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078527.473306:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078527.665168:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078528.177543:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078528.177775:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078528.178644:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078528.241205:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078528.500611:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078528.689164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078529.201169:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078529.201280:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078529.201357:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078529.265330:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078529.521179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078529.713180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078530.225196:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078530.226415:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078530.226710:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078530.289348:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078530.545324:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078530.737176:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078531.249149:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078531.249149:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078531.249149:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078531.313524:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078531.569162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078531.569165:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078531.569172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078531.569177:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078531.569186:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4237 00000100:00080000:3.0:1769078531.569196:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078531.569203:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078531.569239:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913657856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078531.569262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078531.569266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078531.569268:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078531.569276:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4237 00000100:00080000:3.0:1769078531.569280:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078531.569284:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078531.569302:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078531.569308:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913657856:192.168.204.121@tcp:250:kworker.0 00000100:00100000:3.0:1769078531.569313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913657984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078531.569322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078531.569325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078531.569327:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078531.569331:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4237 00000100:00080000:3.0:1769078531.569333:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078531.569337:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078531.569373:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078531.569386:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913658112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078531.569387:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078531.569399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078531.569402:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078531.569403:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078531.569407:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4237 00000100:00080000:3.0:1769078531.569410:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078531.569414:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078531.569418:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078531.569424:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078531/real 1769078531] req@ffff935002a7ca80 x1855008913657856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078531.569427:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913658240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078531.569435:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078531.569439:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078531.569441:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078531.569445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4237 00000100:00080000:3.0:1769078531.569447:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078531.569451:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078531.569452:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913657856/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078531.569455:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769078531.569462:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078531.569463:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913658368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078531.569466:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078531.569470:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913657856:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078531.569472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078531.569475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078531.569477:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078531.569480:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4237 00000100:00080000:3.0:1769078531.569483:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078531.569486:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078531.569490:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078531.569491:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:3.0:1769078531.569498:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913658496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078531.569506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078531.569510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078531.569511:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078531.569514:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4237 00000100:00080000:3.0:1769078531.569518:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078531.569519:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:3.0:1769078531.569521:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078531.569525:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078531.569530:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078531/real 1769078531] req@ffff935002a7f100 x1855008913657984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078531.569535:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913658624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078531.569542:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913657984/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078531.569550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078531.569553:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078531.569556:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913657984:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078531.569561:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913658112:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078531.569580:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078531.569587:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078531/real 1769078531] req@ffff935002a7df80 x1855008913658112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078531.569594:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913658112/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078531.569603:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078531.569605:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078531.569607:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913658112:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078531.569610:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913658240:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078531.569619:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078531.569623:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078531/real 1769078531] req@ffff935002a7f800 x1855008913658240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078531.569630:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913658240/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078531.569634:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078531.569636:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078531.569637:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913658240:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078531.569640:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913658368:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078531.569649:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078531.569653:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078531/real 1769078531] req@ffff935002a7c000 x1855008913658368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078531.569660:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913658368/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078531.569664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078531.569666:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078531.569668:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913658368:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078531.569680:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913658496:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078531.569689:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078531.569694:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078531/real 1769078531] req@ffff935002a7ca80 x1855008913658496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078531.569700:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913658496/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078531.569704:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078531.569706:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078531.569707:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913658496:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078531.569710:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913658624:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078531.569718:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078531.569722:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078531/real 1769078531] req@ffff935002a7c700 x1855008913658624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078531.569729:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913658624/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078531.569733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078531.569735:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078531.569736:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913658624:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078531.763373:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078532.273345:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078532.273787:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078532.274419:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078532.337475:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078532.593166:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078532.785172:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078533.297157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078533.297157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078533.297459:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078533.361682:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078533.617173:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078533.809162:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078534.321171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078534.321250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078534.321346:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078534.385255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078534.641645:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078534.833216:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078535.345171:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078535.345250:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078535.345370:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078535.409157:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078535.665199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078535.857427:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078536.369164:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078536.369164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078536.369164:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078536.433237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078536.689363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078536.689376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078536.689381:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078536.689393:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4242 00000100:00080000:1.0:1769078536.689405:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078536.689412:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078536.689460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913658752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078536.689547:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913658752:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769078536.689631:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078536.689666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078536.689670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078536.689672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078536.689677:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4242 00000100:00080000:1.0:1769078536.689681:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078536.689685:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078536.689695:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078536.689709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913658880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078536.689785:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:0.0:1769078536.690368:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078536/real 1769078536] req@ffff9350100c9c00 x1855008913658752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078536.690391:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913658752/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078536.690401:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078536.690405:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078536.690410:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913658752:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078536.690435:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913658880:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078536.690455:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078536.690464:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078536/real 1769078536] req@ffff9350100c9500 x1855008913658880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078536.690472:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913658880/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078536.690477:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078536.690480:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078536.690482:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913658880:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078536.691371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078536.691377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078536.691380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078536.691387:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4242 00000100:00080000:1.0:1769078536.691393:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078536.691399:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078536.691409:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078536.691431:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913659008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078536.691456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078536.691460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078536.691462:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078536.691467:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4242 00000100:00080000:1.0:1769078536.691470:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078536.691473:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078536.691478:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078536.691489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913659136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078536.691499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078536.691502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078536.691504:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078536.691507:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4242 00000100:00080000:1.0:1769078536.691511:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078536.691515:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078536.691520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078536.691529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913659264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078536.691543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078536.691546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078536.691548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078536.691552:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4242 00000100:00080000:1.0:1769078536.691555:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078536.691558:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078536.691562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078536.691573:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913659392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078536.691581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078536.691584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078536.691586:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078536.691589:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4242 00000100:00080000:1.0:1769078536.691592:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078536.691596:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078536.691601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078536.691610:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913659520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078536.691697:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659008:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078536.691746:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078536.691764:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078536/real 1769078536] req@ffff9350100caa00 x1855008913659008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078536.691777:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913659008/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078536.691790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078536.691794:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078536.691797:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659008:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078536.691802:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659136:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078536.691822:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078536.691833:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078536/real 1769078536] req@ffff9350100c8380 x1855008913659136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078536.691843:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913659136/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078536.691849:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078536.691852:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078536.691855:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659136:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078536.691860:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913659264:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078536.691877:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078536.691889:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078536/real 1769078536] req@ffff9350100ca680 x1855008913659264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078536.691899:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913659264/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078536.691907:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078536.691910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078536.691913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913659264:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078536.691920:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913659392:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078536.691936:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078536.691946:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078536/real 1769078536] req@ffff9350100c8e00 x1855008913659392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078536.691956:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913659392/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078536.691963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078536.691965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078536.691968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913659392:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078536.691973:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913659520:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078536.691988:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078536.691998:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078536/real 1769078536] req@ffff9350100ca300 x1855008913659520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078536.692007:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913659520/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078536.692014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078536.692017:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078536.692020:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913659520:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078536.881309:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078537.393195:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078537.393195:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078537.393338:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078537.457215:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078537.713179:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078537.905164:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078538.417229:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078538.417229:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078538.417229:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078538.481268:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078538.737203:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078538.929154:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078539.441170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078539.441170:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078539.441170:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078539.505189:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078539.761167:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078539.953175:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078540.465252:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078540.465391:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078540.466614:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078540.529225:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078540.785403:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078540.977289:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078541.489190:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078541.489198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078541.490685:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078541.553267:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078541.809191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078541.809203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078541.809207:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078541.809218:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4247 00000100:00080000:1.0:1769078541.809230:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078541.809233:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078541.809242:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078541.809283:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb800 x1855008913659648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078541.809312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078541.809316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078541.809318:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078541.809322:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4247 00000100:00080000:1.0:1769078541.809326:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078541.809329:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078541.809340:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078541.809355:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca300 x1855008913659776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078541.809358:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913659648:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078541.809375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078541.809379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078541.809381:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078541.809386:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4247 00000100:00080000:1.0:1769078541.809390:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078541.809395:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078541.809400:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078541.809411:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8e00 x1855008913659904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078541.809421:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078541.809422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078541.809427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078541.809428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078541.809434:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4247 00000100:00080000:1.0:1769078541.809438:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078541.809442:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078541.809447:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078541.809459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100ca680 x1855008913660032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078541.809462:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078541/real 1769078541] req@ffff9350100cb800 x1855008913659648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078541.809478:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb800 x1855008913659648/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078541.809483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078541.809492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078541.809496:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078541.809498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078541.809502:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4247 00000100:00080000:0.0:1769078541.809508:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:1.0:1769078541.809509:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078541.809516:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913659648:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078541.809517:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078541.809525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078541.809538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8380 x1855008913660160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078541.809547:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659776:192.168.204.121@tcp:38:kworker.0 00000100:00080000:1.0:1769078541.809563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078541.809568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078541.809570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078541.809575:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4247 00000100:00080000:1.0:1769078541.809582:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078541.809589:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:1.0:1769078541.809590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078541.809599:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078541.809610:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078541/real 1769078541] req@ffff9350100ca300 x1855008913659776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078541.809613:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100caa00 x1855008913660288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078541.809636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078541.809640:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca300 x1855008913659776/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078541.809641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078541.809644:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078541.809648:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4247 00000100:00080000:1.0:1769078541.809652:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078541.809659:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00080000:0.0:1769078541.809668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769078541.809669:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769078541.809674:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078541.809678:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659776:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078541.809686:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078541.809686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9500 x1855008913660416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078541.809730:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078541.809743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078541/real 1769078541] req@ffff9350100c8e00 x1855008913659904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078541.809756:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8e00 x1855008913659904/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078541.809764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078541.809767:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078541.809769:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913659904:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078541.809773:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660032:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078541.809788:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078541.809795:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078541/real 1769078541] req@ffff9350100ca680 x1855008913660032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078541.809813:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100ca680 x1855008913660032/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078541.809821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078541.809824:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078541.809827:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660032:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078541.809850:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913660160:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078541.809866:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078541.809874:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078541/real 1769078541] req@ffff9350100c8380 x1855008913660160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078541.809898:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8380 x1855008913660160/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078541.809906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078541.809909:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078541.809912:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913660160:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078541.809918:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913660288:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078541.809937:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078541.809954:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078541/real 1769078541] req@ffff9350100caa00 x1855008913660288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078541.809966:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100caa00 x1855008913660288/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078541.809975:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078541.809978:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078541.809981:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913660288:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078541.809987:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913660416:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078541.810005:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078541.810015:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078541/real 1769078541] req@ffff9350100c9500 x1855008913660416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078541.810027:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9500 x1855008913660416/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078541.810041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078541.810044:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078541.810046:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913660416:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078542.001122:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078542.513134:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078542.513153:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078542.513153:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078542.577279:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078542.833810:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078543.025208:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078543.537167:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078543.537167:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078543.537410:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078543.601260:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078543.857191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078544.049196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078544.561210:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078544.561211:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078544.561341:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078544.625345:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078544.881224:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078545.073197:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078545.585333:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078545.585401:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078545.593597:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078545.649211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078545.905199:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078546.100204:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078546.609171:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078546.609171:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078546.609346:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078546.673238:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078546.929169:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078546.929418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078546.929425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078546.929429:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078546.929448:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4252 00000100:00080000:2.0:1769078546.929459:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078546.929466:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078546.929511:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a63100 x1855008913660544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078546.929539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078546.929543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078546.929545:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078546.929549:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4252 00000100:00080000:2.0:1769078546.929561:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078546.929566:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078546.929575:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078546.929587:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61500 x1855008913660672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078546.929595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078546.929598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078546.929599:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078546.929603:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4252 00000100:00080000:2.0:1769078546.929606:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078546.929609:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078546.929614:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078546.929625:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60000 x1855008913660800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078546.929633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078546.929636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078546.929637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078546.929640:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4252 00000100:00080000:2.0:1769078546.929643:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078546.929646:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078546.929650:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078546.929659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62a00 x1855008913660928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078546.929667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078546.929670:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078546.929672:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078546.929675:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4252 00000100:00080000:2.0:1769078546.929678:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078546.929681:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078546.929685:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078546.929696:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913661056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078546.929705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078546.929708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078546.929710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078546.929713:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4252 00000100:00080000:2.0:1769078546.929716:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078546.929720:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078546.929725:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078546.929735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913661184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078546.930138:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913660544:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769078546.930394:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078546.930436:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078546/real 1769078546] req@ffff935002a63100 x1855008913660544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078546.930453:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a63100 x1855008913660544/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078546.930463:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078546.930467:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078546.930471:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913660544:192.168.204.121@tcp:250:kworker.0 00000100:00080000:2.0:1769078546.930570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078546.930573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078546.930575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078546.930579:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4252 00000100:00080000:2.0:1769078546.930583:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078546.930587:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078546.930593:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078546.930605:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913661312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078546.930715:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660672:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078546.931020:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078546.931031:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078546/real 1769078546] req@ffff935002a61500 x1855008913660672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078546.931040:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61500 x1855008913660672/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078546.931047:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078546.931069:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078546.931071:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660672:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078546.931076:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660800:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078546.931101:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078546.931248:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078546/real 1769078546] req@ffff935002a60000 x1855008913660800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078546.931264:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60000 x1855008913660800/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078546.931332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078546.931336:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078546.931340:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660800:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078546.931347:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660928:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078546.931369:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078546.931379:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078546/real 1769078546] req@ffff935002a62a00 x1855008913660928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078546.931391:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62a00 x1855008913660928/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078546.931399:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078546.931401:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078546.931405:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913660928:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078546.931411:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661056:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078546.931428:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078546.931542:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078546/real 1769078546] req@ffff935002a61c00 x1855008913661056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078546.931557:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913661056/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078546.931565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078546.931569:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078546.931572:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661056:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078546.931599:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661184:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078546.931617:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078546.931627:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078546/real 1769078546] req@ffff935002a61f80 x1855008913661184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078546.931639:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913661184/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078546.931647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078546.931650:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078546.931653:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661184:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078546.931658:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661312:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078546.931671:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078546.931680:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078546/real 1769078546] req@ffff935002a60a80 x1855008913661312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078546.931690:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913661312/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078546.931697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078546.931700:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078546.931703:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661312:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078547.121205:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078547.633267:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078547.633267:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078547.633268:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078547.697511:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078547.953476:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078548.145225:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078548.657198:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078548.657201:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078548.657217:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078548.721170:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078548.977208:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078549.169186:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078549.681232:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078549.681232:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078549.681279:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078549.745214:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078550.001214:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078550.193180:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078550.705184:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078550.705404:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078550.705617:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078550.769161:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078551.025256:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078551.217117:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078551.729155:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078551.729155:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078551.729240:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078551.793203:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078552.049641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078552.049649:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:1.0:1769078552.049654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078552.049659:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078552.049668:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4257 00000100:00080000:1.0:1769078552.049679:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078552.049685:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:1.0:1769078552.049720:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9c00 x1855008913661440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078552.049746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078552.049750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078552.049752:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078552.049756:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4257 00000100:00080000:1.0:1769078552.049760:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078552.049764:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078552.049773:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078552.049784:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cb100 x1855008913661568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078552.049794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078552.049797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078552.049799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078552.049805:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4257 00000100:00100000:2.0:1769078552.049805:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913661440:192.168.204.121@tcp:250:kworker.0 00000100:00080000:1.0:1769078552.049811:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078552.049816:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078552.049822:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078552.049835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9180 x1855008913661696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078552.049845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078552.049848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078552.049849:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078552.049853:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4257 00000100:00080000:1.0:1769078552.049856:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078552.049860:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078552.049864:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1769078552.049874:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8a80 x1855008913661824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078552.049882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078552.049885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078552.049887:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078552.049891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4257 00000100:00080000:1.0:1769078552.049894:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078552.049911:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078552.049915:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078552.049924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c8000 x1855008913661952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078552.049933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078552.049936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078552.049938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078552.049941:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4257 00000100:00080000:1.0:1769078552.049943:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078552.049947:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078552.049950:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078552.049959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100c9880 x1855008913662080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078552.049967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078552.049970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078552.049972:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078552.049975:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4257 00000100:00080000:1.0:1769078552.049979:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078552.049982:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:1.0:1769078552.049986:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078552.049994:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350100cad80 x1855008913662208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078552.050085:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078552.050120:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078552/real 1769078552] req@ffff9350100c9c00 x1855008913661440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078552.050138:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9c00 x1855008913661440/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078552.050147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078552.050151:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078552.050154:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913661440:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078552.050161:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913661568:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078552.050182:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078552.050204:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078552/real 1769078552] req@ffff9350100cb100 x1855008913661568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078552.050215:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cb100 x1855008913661568/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078552.050224:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078552.050227:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078552.050230:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913661568:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078552.050371:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913661696:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078552.050449:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078552.050460:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078552/real 1769078552] req@ffff9350100c9180 x1855008913661696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078552.050477:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9180 x1855008913661696/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078552.050485:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078552.050487:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078552.050490:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913661696:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078552.050496:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913661824:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078552.050510:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078552.050518:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078552/real 1769078552] req@ffff9350100c8a80 x1855008913661824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078552.050529:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8a80 x1855008913661824/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078552.050536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078552.050539:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078552.050542:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913661824:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078552.050546:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661952:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078552.050558:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078552.050565:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078552/real 1769078552] req@ffff9350100c8000 x1855008913661952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078552.050574:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c8000 x1855008913661952/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078552.050587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078552.050590:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078552.050593:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913661952:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078552.050598:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662080:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078552.050612:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078552.050620:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078552/real 1769078552] req@ffff9350100c9880 x1855008913662080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078552.050630:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100c9880 x1855008913662080/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078552.050638:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078552.050641:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078552.050643:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662080:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078552.050648:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662208:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078552.050663:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078552.050671:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078552/real 1769078552] req@ffff9350100cad80 x1855008913662208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078552.050682:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350100cad80 x1855008913662208/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078552.050689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078552.050691:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078552.050695:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350100cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662208:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078552.241166:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078552.753157:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078552.753157:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078552.753157:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078552.817163:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078553.073191:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078553.265127:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078553.777152:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078553.777152:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:3.0:1769078553.777152:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078553.841221:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078554.097197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078554.289161:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078554.801164:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078554.801165:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078554.801165:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078554.865157:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078555.121126:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078555.313121:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078555.825135:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078555.825135:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078555.825143:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078555.889180:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078556.145151:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078556.337144:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078556.849227:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078556.849868:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078556.851359:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078556.913208:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078557.169726:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:0.0:1769078557.169867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078557.169874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078557.169878:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078557.169887:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4262 00000100:00080000:0.0:1769078557.169897:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078557.169903:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:0.0:1769078557.169954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913662336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078557.169988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078557.169991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078557.169993:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078557.169997:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4262 00000100:00080000:0.0:1769078557.170001:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078557.170006:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078557.170016:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078557.170029:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9500 x1855008913662464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078557.170034:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913662336:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078557.170044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078557.170074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078557.170076:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078557.170082:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4262 00000100:00080000:0.0:1769078557.170085:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078557.170089:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078557.170095:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078557.170108:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fbb80 x1855008913662592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078557.170118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078557.170122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078557.170123:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078557.170128:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4262 00000100:00080000:0.0:1769078557.170132:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078557.170136:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000800:00000100:2.0:1769078557.170141:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000008:00100000:0.0:1769078557.170144:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1769078557.170159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9c00 x1855008913662720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078557.170169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078557.170173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078557.170175:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078557.170180:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4262 00000100:00100000:2.0:1769078557.170184:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078557/real 1769078557] req@ffff9350103f9f80 x1855008913662336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078557.170186:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078557.170194:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078557.170202:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078557.170212:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913662336/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078557.170217:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9180 x1855008913662848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078557.170235:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078557.170237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078557.170242:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078557.170244:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00080000:0.0:1769078557.170245:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078557.170251:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913662336:192.168.204.121@tcp:250:kworker.0 00000100:00080000:0.0:1769078557.170252:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4262 00000100:00080000:0.0:1769078557.170259:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078557.170264:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078557.170268:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078557.170279:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103f9f80 x1855008913662976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078557.170280:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913662464:192.168.204.121@tcp:38:kworker.0 00000100:00080000:0.0:1769078557.170302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078557.170306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078557.170308:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078557.170313:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4262 00000100:00080000:0.0:1769078557.170318:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078557.170323:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00080000:0.0:1769078557.170325:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:0.0:1769078557.170332:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078557.170339:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078557/real 1769078557] req@ffff9350103f9500 x1855008913662464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078557.170346:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9350103fad80 x1855008913663104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078557.170357:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9500 x1855008913662464/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078557.170367:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078557.170372:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078557.170376:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913662464:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078557.170383:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913662592:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078557.170399:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078557.170407:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078557/real 1769078557] req@ffff9350103fbb80 x1855008913662592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078557.170417:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fbb80 x1855008913662592/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078557.170425:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078557.170428:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078557.170431:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913662592:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078557.170436:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913662720:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078557.170449:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078557.170457:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078557/real 1769078557] req@ffff9350103f9c00 x1855008913662720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078557.170468:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9c00 x1855008913662720/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078557.170475:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078557.170478:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078557.170481:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913662720:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078557.170486:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662848:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078557.170499:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078557.170506:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078557/real 1769078557] req@ffff9350103f9180 x1855008913662848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078557.170516:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9180 x1855008913662848/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078557.170523:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078557.170526:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078557.170529:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662848:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078557.170543:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662976:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078557.170556:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078557.170564:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078557/real 1769078557] req@ffff9350103f9f80 x1855008913662976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078557.170574:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103f9f80 x1855008913662976/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078557.170581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078557.170584:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078557.170588:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913662976:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078557.170593:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913663104:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078557.170604:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078557.170612:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078557/real 1769078557] req@ffff9350103fad80 x1855008913663104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078557.170623:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9350103fad80 x1855008913663104/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078557.170630:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078557.170632:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078557.170635:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9350103fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913663104:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078557.361142:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078557.873166:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078557.873169:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078557.873228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078557.937437:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078558.193138:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078558.385165:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078558.897161:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078558.897162:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078558.897192:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078558.961156:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078559.217215:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078559.409207:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078559.921447:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078559.921577:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078559.921896:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078559.985255:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078560.243012:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078560.433177:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078560.945202:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078560.945202:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078560.945208:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078561.009263:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078561.265132:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078561.458010:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078561.969267:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078561.969291:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078561.969403:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078562.033190:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078562.289197:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078562.289197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078562.289209:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078562.289214:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078562.289224:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4267 00000100:00080000:3.0:1769078562.289234:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078562.289241:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078562.289281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7fb80 x1855008913663232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078562.289305:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078562.289309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078562.289311:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078562.289315:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4267 00000100:00080000:3.0:1769078562.289318:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078562.289322:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078562.289334:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078562.289346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7d180 x1855008913663360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078562.289362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078562.289365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078562.289367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078562.289371:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4267 00000100:00080000:3.0:1769078562.289374:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078562.289378:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078562.289382:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078562.289392:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7e300 x1855008913663488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078562.289400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078562.289403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078562.289404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078562.289408:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4267 00000100:00080000:3.0:1769078562.289410:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078562.289413:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913663232:192.168.204.121@tcp:250:kworker.0 00000100:00080000:3.0:1769078562.289414:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078562.289418:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078562.289427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c380 x1855008913663616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078562.289434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078562.289437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078562.289438:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078562.289441:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4267 00000100:00080000:3.0:1769078562.289445:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078562.289448:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078562.289452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078562.289460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913663744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078562.289467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078562.289470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078562.289471:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078562.289474:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4267 00000100:00080000:3.0:1769078562.289483:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078562.289486:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078562.289490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078562.289494:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:3.0:1769078562.289499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913663872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078562.289506:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078562.289509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078562.289510:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078562.289513:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4267 00000100:00080000:3.0:1769078562.289516:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078562.289519:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078562.289523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078562.289530:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913664000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078562.289539:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078562/real 1769078562] req@ffff935002a7fb80 x1855008913663232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078562.289556:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7fb80 x1855008913663232/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078562.289566:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078562.289571:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078562.289575:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913663232:192.168.204.121@tcp:250:kworker.0 00000100:00100000:2.0:1769078562.289582:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913663360:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078562.289600:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078562.289609:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078562/real 1769078562] req@ffff935002a7d180 x1855008913663360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078562.289617:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7d180 x1855008913663360/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078562.289623:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078562.289625:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078562.289627:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913663360:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078562.289631:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913663488:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078562.289641:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078562.289646:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078562/real 1769078562] req@ffff935002a7e300 x1855008913663488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078562.289653:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7e300 x1855008913663488/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078562.289657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078562.289659:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078562.289661:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913663488:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078562.289684:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913663616:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078562.289693:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078562.289699:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078562/real 1769078562] req@ffff935002a7c380 x1855008913663616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078562.289706:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c380 x1855008913663616/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078562.289711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078562.289712:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078562.289714:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913663616:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078562.289717:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913663744:192.168.204.121@tcp:38:kworker.0 00000800:00000100:2.0:1769078562.289725:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078562.289731:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078562/real 1769078562] req@ffff935002a7dc00 x1855008913663744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078562.289737:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913663744/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078562.289742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078562.289743:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078562.289745:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913663744:192.168.204.121@tcp:38:kworker.0 00000100:00100000:2.0:1769078562.289748:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913663872:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078562.289755:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078562.289760:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078562/real 1769078562] req@ffff935002a7c700 x1855008913663872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078562.289766:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913663872/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078562.289771:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078562.289773:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078562.289775:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913663872:192.168.204.121@tcp:8:kworker.0 00000100:00100000:2.0:1769078562.289779:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664000:192.168.204.121@tcp:8:kworker.0 00000800:00000100:2.0:1769078562.289793:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:2.0:1769078562.289802:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078562/real 1769078562] req@ffff935002a7ca80 x1855008913664000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078562.289812:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913664000/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078562.289820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078562.289823:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:2.0:1769078562.289826:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664000:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078562.481152:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078562.993101:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078562.993109:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078562.993113:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078563.057199:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078563.313145:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078563.505187:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078564.017185:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078564.017185:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078564.017186:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078564.081237:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078564.337200:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078564.529191:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078565.041193:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078565.041293:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078565.041709:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078565.110612:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078565.362735:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078565.553145:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078566.065237:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078566.068126:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078566.068910:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078566.129448:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078566.385717:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:1.0:1769078566.577696:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078567.089204:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078567.091984:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078567.093608:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078567.153354:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078567.409594:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:2.0:1769078567.412337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078567.412348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078567.412352:0:37:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078567.412362:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4273 00000100:00080000:2.0:1769078567.412373:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078567.412380:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:2.0:1769078567.412415:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62680 x1855008913664128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078567.412451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078567.412455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078567.412456:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078567.412461:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4273 00000100:00080000:2.0:1769078567.412465:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078567.412468:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078567.412478:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078567.412490:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61180 x1855008913664256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078567.412499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078567.412503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078567.412505:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078567.412508:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4273 00000100:00080000:2.0:1769078567.412511:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078567.412515:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078567.412519:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078567.412529:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61880 x1855008913664384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078567.412542:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078567.412545:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078567.412547:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078567.412550:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4273 00000100:00080000:2.0:1769078567.412552:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078567.412555:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078567.412559:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1769078567.412569:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a62300 x1855008913664512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078567.412577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078567.412580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078567.412582:0:37:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078567.412585:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4273 00000100:00080000:2.0:1769078567.412588:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078567.412591:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078567.412595:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078567.412604:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a60a80 x1855008913664640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078567.412613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078567.412616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078567.412617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078567.412620:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4273 00000100:00080000:2.0:1769078567.412623:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078567.412626:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078567.412630:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078567.412638:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61f80 x1855008913664768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078567.412646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078567.412649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078567.412651:0:37:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078567.412654:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4273 00000100:00080000:2.0:1769078567.412656:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078567.412659:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:2.0:1769078567.412662:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078567.412671:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a61c00 x1855008913664896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078567.414691:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913664128:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769078567.414785:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078567.414829:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078567/real 1769078567] req@ffff935002a62680 x1855008913664128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078567.414847:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62680 x1855008913664128/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078567.414862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078567.414867:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078567.414871:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913664128:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078567.414878:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913664256:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078567.414901:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078567.414913:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078567/real 1769078567] req@ffff935002a61180 x1855008913664256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078567.414924:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61180 x1855008913664256/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078567.414932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078567.414935:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078567.414937:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913664256:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078567.414943:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913664384:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078567.414961:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078567.414970:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078567/real 1769078567] req@ffff935002a61880 x1855008913664384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078567.414980:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61880 x1855008913664384/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078567.414988:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078567.414991:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078567.414994:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913664384:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078567.415000:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913664512:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078567.415017:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078567.415027:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078567/real 1769078567] req@ffff935002a62300 x1855008913664512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078567.415038:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a62300 x1855008913664512/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078567.415046:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078567.415084:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078567.415088:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913664512:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078567.415095:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664640:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078567.415115:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078567.415125:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078567/real 1769078567] req@ffff935002a60a80 x1855008913664640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078567.415135:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a60a80 x1855008913664640/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078567.415142:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078567.415145:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078567.415148:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664640:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078567.415154:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664768:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078567.415171:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078567.415181:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078567/real 1769078567] req@ffff935002a61f80 x1855008913664768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078567.415192:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61f80 x1855008913664768/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078567.415200:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078567.415203:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078567.415206:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664768:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078567.415212:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664896:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078567.415474:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078567.415486:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078567/real 1769078567] req@ffff935002a61c00 x1855008913664896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078567.415498:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a61c00 x1855008913664896/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078567.415507:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078567.415511:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078567.415514:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913664896:192.168.204.121@tcp:8:kworker.0 00000100:00100000:1.0:1769078567.601203:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078568.113192:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078568.113300:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078568.113379:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078568.177241:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078568.435072:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078568.630842:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078569.137176:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078569.137180:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078569.137737:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00010000:00010000:2.0:1769078569.201211:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078569.457875:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078569.649500:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078570.161309:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078570.161399:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078570.161438:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078570.225187:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078570.481184:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078570.673996:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078571.185170:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078571.185200:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078571.185279:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078571.249212:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078571.505160:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078571.697212:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078572.209188:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078572.209189:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078572.209307:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078572.275797:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078572.529315:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00080000:3.0:1769078572.532177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 987cd90b-10bf-40f3-bb36-40b816fac7c1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078572.532189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078572.532194:0:8:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078572.532205:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.121@tcp: connect to NID 192.168.204.121@tcp last attempt 4278 00000100:00080000:3.0:1769078572.532217:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.121@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078572.532224:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.121@tcp: import ffff935007dde000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000100:00100000:3.0:1769078572.532263:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c000 x1855008913665024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078572.532296:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078572.532300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078572.532302:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078572.532306:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4278 00000100:00080000:3.0:1769078572.532310:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078572.532314:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9350067ee000: import ffff9350052ab800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078572.532325:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078572.532337:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f800 x1855008913665152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078572.532347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078572.532350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078572.532352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078572.532356:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4278 00000100:00080000:3.0:1769078572.532368:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078572.532373:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9350067ee000: import ffff9350052ac000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078572.532378:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078572.532388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7df80 x1855008913665280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078572.532398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7afd476b-94ee-4c94-b080-d15824f3ae56->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078572.532401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078572.532403:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078572.532406:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: connect to NID 192.168.204.121@tcp last attempt 4278 00000100:00080000:3.0:1769078572.532409:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078572.532412:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9350067ee000: import ffff9350052aa800 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078572.532416:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1769078572.532425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7f100 x1855008913665408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078572.532434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078572.532437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078572.532439:0:8:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078572.532442:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4278 00000100:00080000:3.0:1769078572.532445:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078572.532449:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff935006f53800: import ffff935008fdb000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078572.532453:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078572.532466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7ca80 x1855008913665536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078572.532475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078572.532478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078572.532479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078572.532483:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4278 00000100:00080000:3.0:1769078572.532485:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078572.532488:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff935006f53800: import ffff9350052d8000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078572.532493:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078572.532502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7c700 x1855008913665664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078572.532510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84af2899-c956-435d-9e3c-5d5fbc3c9246->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078572.532513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078572.532514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078572.532517:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: connect to NID 192.168.204.121@tcp last attempt 4278 00000100:00080000:3.0:1769078572.532520:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078572.532524:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff935006f53800: import ffff9350052d9000 using connection 192.168.204.121@tcp/192.168.204.121@tcp 00000008:00100000:3.0:1769078572.532528:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681347 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078572.532537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff935002a7dc00 x1855008913665792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078572.532623:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913665024:192.168.204.121@tcp:250:kworker.0 00000800:00000100:0.0:1769078572.532701:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078572.532743:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078572/real 1769078572] req@ffff935002a7c000 x1855008913665024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078572.532760:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c000 x1855008913665024/t0(0) o250->MGC192.168.204.121@tcp@192.168.204.121@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078572.532770:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935007dde000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078572.532775:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078572.532778:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:987cd90b-10bf-40f3-bb36-40b816fac7c1:2361:1855008913665024:192.168.204.121@tcp:250:kworker.0 00000100:00100000:0.0:1769078572.532787:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913665152:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078572.532809:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078572.532820:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078572/real 1769078572] req@ffff935002a7f800 x1855008913665152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078572.532832:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f800 x1855008913665152/t0(0) o38->lustre-MDT0000-mdc-ffff9350067ee000@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078572.532840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ab800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078572.532844:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078572.532847:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913665152:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078572.532853:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913665280:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078572.532871:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078572.532881:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078572/real 1769078572] req@ffff935002a7df80 x1855008913665280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078572.532893:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7df80 x1855008913665280/t0(0) o8->lustre-OST0000-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078572.532906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052ac000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078572.532910:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078572.532913:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913665280:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078572.532919:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913665408:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078572.532935:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078572.532945:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078572/real 1769078572] req@ffff935002a7f100 x1855008913665408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078572.532954:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7f100 x1855008913665408/t0(0) o8->lustre-OST0001-osc-ffff9350067ee000@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078572.532962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052aa800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078572.532965:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078572.532968:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7afd476b-94ee-4c94-b080-d15824f3ae56:2361:1855008913665408:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078572.532972:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913665536:192.168.204.121@tcp:38:kworker.0 00000800:00000100:0.0:1769078572.532989:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078572.532998:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078572/real 1769078572] req@ffff935002a7ca80 x1855008913665536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078572.533007:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7ca80 x1855008913665536/t0(0) o38->lustre-MDT0000-mdc-ffff935006f53800@192.168.204.121@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078572.533014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff935008fdb000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078572.533017:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078572.533021:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913665536:192.168.204.121@tcp:38:kworker.0 00000100:00100000:0.0:1769078572.533039:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913665664:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078572.533088:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078572.533100:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078572/real 1769078572] req@ffff935002a7c700 x1855008913665664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078572.533110:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7c700 x1855008913665664/t0(0) o8->lustre-OST0000-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078572.533117:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d8000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078572.533120:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078572.533123:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913665664:192.168.204.121@tcp:8:kworker.0 00000100:00100000:0.0:1769078572.533128:0:2361:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913665792:192.168.204.121@tcp:8:kworker.0 00000800:00000100:0.0:1769078572.533144:0:2361:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.121@tcp 00000100:00100000:0.0:1769078572.533154:0:2361:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078572/real 1769078572] req@ffff935002a7dc00 x1855008913665792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078572.533165:0:2361:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff935002a7dc00 x1855008913665792/t0(0) o8->lustre-OST0001-osc-ffff935006f53800@192.168.204.121@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078572.533173:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9350052d9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078572.533176:0:2361:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.121@tcp failed (-110) 00000100:00100000:0.0:1769078572.533178:0:2361:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff935002a7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84af2899-c956-435d-9e3c-5d5fbc3c9246:2361:1855008913665792:192.168.204.121@tcp:8:kworker.0 00000100:00100000:3.0:1769078572.721139:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078573.233154:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078573.234025:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078573.235720:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078573.297531:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078573.553302:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078573.746327:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078574.257182:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078574.257417:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078574.257481:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078574.321503:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078574.577178:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078574.769196:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769078575.281151:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078575.281679:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078575.281868:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078575.345390:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078575.603746:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078575.793323:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds 00000100:00100000:2.0:1769078576.305199:0:25069:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097cc0 going to sleep for 0 seconds 00000100:00100000:1.0:1769078576.305228:0:14429:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078576.305321:0:5555:0:(client.c:2690:ptlrpc_set_wait()) set ffff935007799f00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078576.369635:0:25071:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078576.625289:0:25145:0:(client.c:2690:ptlrpc_set_wait()) set ffff935005097a80 going to sleep for 0 seconds 00000100:00100000:3.0:1769078576.817160:0:24870:0:(client.c:2690:ptlrpc_set_wait()) set ffff935002caa0c0 going to sleep for 0 seconds Debug log: 81316 lines, 81316 kept, 0 dropped, 0 bad.